drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c

   1 /*******************************************************************************
   2
   3   Intel 82599 Virtual Function driver
   4   Copyright(c) 1999 - 2014 Intel Corporation.
   5
   6   This program is free software; you can redistribute it and/or modify it
   7   under the terms and conditions of the GNU General Public License,
   8   version 2, as published by the Free Software Foundation.
   9
  10   This program is distributed in the hope it will be useful, but WITHOUT
  11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  13   more details.
  14
  15   You should have received a copy of the GNU General Public License along with
  16   this program; if not, write to the Free Software Foundation, Inc.,
  17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
  18
  19   The full GNU General Public License is included in this distribution in
  20   the file called "COPYING".
  21
  22   Contact Information:
  23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
  24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  25
  26 *******************************************************************************/
  27
  28
  29 /******************************************************************************
  30  Copyright (c)2006 - 2007 Myricom, Inc. for some LRO specific code
  31 ******************************************************************************/
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include <linux/types.h>
  36 #include <linux/bitops.h>
  37 #include <linux/module.h>
  38 #include <linux/pci.h>
  39 #include <linux/netdevice.h>
  40 #include <linux/vmalloc.h>
  41 #include <linux/string.h>
  42 #include <linux/in.h>
  43 #include <linux/ip.h>
  44 #include <linux/tcp.h>
  45 #include <linux/sctp.h>
  46 #include <linux/ipv6.h>
  47 #include <linux/slab.h>
  48 #include <net/checksum.h>
  49 #include <net/ip6_checksum.h>
  50 #include <linux/ethtool.h>
  51 #include <linux/if.h>
  52 #include <linux/if_vlan.h>
  53 #include <linux/prefetch.h>
  54
  55 #include "ixgbevf.h"
  56
  57 const char ixgbevf_driver_name[] = "ixgbevf";
  58 static const char ixgbevf_driver_string[] =
  59         "Intel(R) 10 Gigabit PCI Express Virtual Function Network Driver";
  60
  61 #define DRV_VERSION "2.12.1-k"
  62 const char ixgbevf_driver_version[] = DRV_VERSION;
  63 static char ixgbevf_copyright[] =
  64         "Copyright (c) 2009 - 2012 Intel Corporation.";
  65
  66 static const struct ixgbevf_info *ixgbevf_info_tbl[] = {
  67         [board_82599_vf] = &ixgbevf_82599_vf_info,
  68         [board_X540_vf]  = &ixgbevf_X540_vf_info,
  69         [board_X550_vf]  = &ixgbevf_X550_vf_info,
  70         [board_X550EM_x_vf] = &ixgbevf_X550EM_x_vf_info,
  71 };
  72
  73 /* ixgbevf_pci_tbl - PCI Device ID Table
  74  *
  75  * Wildcard entries (PCI_ANY_ID) should come last
  76  * Last entry must be all 0s
  77  *
  78  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  79  *   Class, Class Mask, private data (not used) }
  80  */
  81 static const struct pci_device_id ixgbevf_pci_tbl[] = {
  82         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_VF), board_82599_vf },
  83         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540_VF), board_X540_vf },
  84         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550_VF), board_X550_vf },
  85         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X550EM_X_VF), board_X550EM_x_vf },
  86         /* required last entry */
  87         {0, }
  88 };
  89 MODULE_DEVICE_TABLE(pci, ixgbevf_pci_tbl);
  90
  91 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  92 MODULE_DESCRIPTION("Intel(R) 10 Gigabit Virtual Function Network Driver");
  93 MODULE_LICENSE("GPL");
  94 MODULE_VERSION(DRV_VERSION);
  95
  96 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
  97 static int debug = -1;
  98 module_param(debug, int, 0);
  99 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
 100
 101 /* forward decls */
 102 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter);
 103 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector);
 104 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter);
 105
 106 static void ixgbevf_remove_adapter(struct ixgbe_hw *hw)
 107 {
 108         struct ixgbevf_adapter *adapter = hw->back;
 109
 110         if (!hw->hw_addr)
 111                 return;
 112         hw->hw_addr = NULL;
 113         dev_err(&adapter->pdev->dev, "Adapter removed\n");
 114         if (test_bit(__IXGBEVF_WORK_INIT, &adapter->state))
 115                 schedule_work(&adapter->watchdog_task);
 116 }
 117
 118 static void ixgbevf_check_remove(struct ixgbe_hw *hw, u32 reg)
 119 {
 120         u32 value;
 121
 122         /* The following check not only optimizes a bit by not
 123          * performing a read on the status register when the
 124          * register just read was a status register read that
 125          * returned IXGBE_FAILED_READ_REG. It also blocks any
 126          * potential recursion.
 127          */
 128         if (reg == IXGBE_VFSTATUS) {
 129                 ixgbevf_remove_adapter(hw);
 130                 return;
 131         }
 132         value = ixgbevf_read_reg(hw, IXGBE_VFSTATUS);
 133         if (value == IXGBE_FAILED_READ_REG)
 134                 ixgbevf_remove_adapter(hw);
 135 }
 136
 137 u32 ixgbevf_read_reg(struct ixgbe_hw *hw, u32 reg)
 138 {
 139         u8 __iomem *reg_addr = ACCESS_ONCE(hw->hw_addr);
 140         u32 value;
 141
 142         if (IXGBE_REMOVED(reg_addr))
 143                 return IXGBE_FAILED_READ_REG;
 144         value = readl(reg_addr + reg);
 145         if (unlikely(value == IXGBE_FAILED_READ_REG))
 146                 ixgbevf_check_remove(hw, reg);
 147         return value;
 148 }
 149
 150 /**
 151  * ixgbevf_set_ivar - set IVAR registers - maps interrupt causes to vectors
 152  * @adapter: pointer to adapter struct
 153  * @direction: 0 for Rx, 1 for Tx, -1 for other causes
 154  * @queue: queue to map the corresponding interrupt to
 155  * @msix_vector: the vector to map to the corresponding queue
 156  */
 157 static void ixgbevf_set_ivar(struct ixgbevf_adapter *adapter, s8 direction,
 158                              u8 queue, u8 msix_vector)
 159 {
 160         u32 ivar, index;
 161         struct ixgbe_hw *hw = &adapter->hw;
 162         if (direction == -1) {
 163                 /* other causes */
 164                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 165                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR_MISC);
 166                 ivar &= ~0xFF;
 167                 ivar |= msix_vector;
 168                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR_MISC, ivar);
 169         } else {
 170                 /* tx or rx causes */
 171                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
 172                 index = ((16 * (queue & 1)) + (8 * direction));
 173                 ivar = IXGBE_READ_REG(hw, IXGBE_VTIVAR(queue >> 1));
 174                 ivar &= ~(0xFF << index);
 175                 ivar |= (msix_vector << index);
 176                 IXGBE_WRITE_REG(hw, IXGBE_VTIVAR(queue >> 1), ivar);
 177         }
 178 }
 179
 180 static void ixgbevf_unmap_and_free_tx_resource(struct ixgbevf_ring *tx_ring,
 181                                         struct ixgbevf_tx_buffer *tx_buffer)
 182 {
 183         if (tx_buffer->skb) {
 184                 dev_kfree_skb_any(tx_buffer->skb);
 185                 if (dma_unmap_len(tx_buffer, len))
 186                         dma_unmap_single(tx_ring->dev,
 187                                          dma_unmap_addr(tx_buffer, dma),
 188                                          dma_unmap_len(tx_buffer, len),
 189                                          DMA_TO_DEVICE);
 190         } else if (dma_unmap_len(tx_buffer, len)) {
 191                 dma_unmap_page(tx_ring->dev,
 192                                dma_unmap_addr(tx_buffer, dma),
 193                                dma_unmap_len(tx_buffer, len),
 194                                DMA_TO_DEVICE);
 195         }
 196         tx_buffer->next_to_watch = NULL;
 197         tx_buffer->skb = NULL;
 198         dma_unmap_len_set(tx_buffer, len, 0);
 199         /* tx_buffer must be completely set up in the transmit path */
 200 }
 201
 202 #define IXGBE_MAX_TXD_PWR       14
 203 #define IXGBE_MAX_DATA_PER_TXD  (1 << IXGBE_MAX_TXD_PWR)
 204
 205 /* Tx Descriptors needed, worst case */
 206 #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD)
 207 #define DESC_NEEDED (MAX_SKB_FRAGS + 4)
 208
 209 static void ixgbevf_tx_timeout(struct net_device *netdev);
 210
 211 /**
 212  * ixgbevf_clean_tx_irq - Reclaim resources after transmit completes
 213  * @q_vector: board private structure
 214  * @tx_ring: tx ring to clean
 215  **/
 216 static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector,
 217                                  struct ixgbevf_ring *tx_ring)
 218 {
 219         struct ixgbevf_adapter *adapter = q_vector->adapter;
 220         struct ixgbevf_tx_buffer *tx_buffer;
 221         union ixgbe_adv_tx_desc *tx_desc;
 222         unsigned int total_bytes = 0, total_packets = 0;
 223         unsigned int budget = tx_ring->count / 2;
 224         unsigned int i = tx_ring->next_to_clean;
 225
 226         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
 227                 return true;
 228
 229         tx_buffer = &tx_ring->tx_buffer_info[i];
 230         tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
 231         i -= tx_ring->count;
 232
 233         do {
 234                 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
 235
 236                 /* if next_to_watch is not set then there is no work pending */
 237                 if (!eop_desc)
 238                         break;
 239
 240                 /* prevent any other reads prior to eop_desc */
 241                 read_barrier_depends();
 242
 243                 /* if DD is not set pending work has not been completed */
 244                 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
 245                         break;
 246
 247                 /* clear next_to_watch to prevent false hangs */
 248                 tx_buffer->next_to_watch = NULL;
 249
 250                 /* update the statistics for this packet */
 251                 total_bytes += tx_buffer->bytecount;
 252                 total_packets += tx_buffer->gso_segs;
 253
 254                 /* free the skb */
 255                 dev_kfree_skb_any(tx_buffer->skb);
 256
 257                 /* unmap skb header data */
 258                 dma_unmap_single(tx_ring->dev,
 259                                  dma_unmap_addr(tx_buffer, dma),
 260                                  dma_unmap_len(tx_buffer, len),
 261                                  DMA_TO_DEVICE);
 262
 263                 /* clear tx_buffer data */
 264                 tx_buffer->skb = NULL;
 265                 dma_unmap_len_set(tx_buffer, len, 0);
 266
 267                 /* unmap remaining buffers */
 268                 while (tx_desc != eop_desc) {
 269                         tx_buffer++;
 270                         tx_desc++;
 271                         i++;
 272                         if (unlikely(!i)) {
 273                                 i -= tx_ring->count;
 274                                 tx_buffer = tx_ring->tx_buffer_info;
 275                                 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
 276                         }
 277
 278                         /* unmap any remaining paged data */
 279                         if (dma_unmap_len(tx_buffer, len)) {
 280                                 dma_unmap_page(tx_ring->dev,
 281                                                dma_unmap_addr(tx_buffer, dma),
 282                                                dma_unmap_len(tx_buffer, len),
 283                                                DMA_TO_DEVICE);
 284                                 dma_unmap_len_set(tx_buffer, len, 0);
 285                         }
 286                 }
 287
 288                 /* move us one more past the eop_desc for start of next pkt */
 289                 tx_buffer++;
 290                 tx_desc++;
 291                 i++;
 292                 if (unlikely(!i)) {
 293                         i -= tx_ring->count;
 294                         tx_buffer = tx_ring->tx_buffer_info;
 295                         tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
 296                 }
 297
 298                 /* issue prefetch for next Tx descriptor */
 299                 prefetch(tx_desc);
 300
 301                 /* update budget accounting */
 302                 budget--;
 303         } while (likely(budget));
 304
 305         i += tx_ring->count;
 306         tx_ring->next_to_clean = i;
 307         u64_stats_update_begin(&tx_ring->syncp);
 308         tx_ring->stats.bytes += total_bytes;
 309         tx_ring->stats.packets += total_packets;
 310         u64_stats_update_end(&tx_ring->syncp);
 311         q_vector->tx.total_bytes += total_bytes;
 312         q_vector->tx.total_packets += total_packets;
 313
 314 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
 315         if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
 316                      (ixgbevf_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
 317                 /* Make sure that anybody stopping the queue after this
 318                  * sees the new next_to_clean.
 319                  */
 320                 smp_mb();
 321
 322                 if (__netif_subqueue_stopped(tx_ring->netdev,
 323                                              tx_ring->queue_index) &&
 324                     !test_bit(__IXGBEVF_DOWN, &adapter->state)) {
 325                         netif_wake_subqueue(tx_ring->netdev,
 326                                             tx_ring->queue_index);
 327                         ++tx_ring->tx_stats.restart_queue;
 328                 }
 329         }
 330
 331         return !!budget;
 332 }
 333
 334 /**
 335  * ixgbevf_rx_skb - Helper function to determine proper Rx method
 336  * @q_vector: structure containing interrupt and ring information
 337  * @skb: packet to send up
 338  **/
 339 static void ixgbevf_rx_skb(struct ixgbevf_q_vector *q_vector,
 340                            struct sk_buff *skb)
 341 {
 342 #ifdef CONFIG_NET_RX_BUSY_POLL
 343         skb_mark_napi_id(skb, &q_vector->napi);
 344
 345         if (ixgbevf_qv_busy_polling(q_vector)) {
 346                 netif_receive_skb(skb);
 347                 /* exit early if we busy polled */
 348                 return;
 349         }
 350 #endif /* CONFIG_NET_RX_BUSY_POLL */
 351
 352         napi_gro_receive(&q_vector->napi, skb);
 353 }
 354
 355 /* ixgbevf_rx_checksum - indicate in skb if hw indicated a good cksum
 356  * @ring: structure containig ring specific data
 357  * @rx_desc: current Rx descriptor being processed
 358  * @skb: skb currently being received and modified
 359  */
 360 static inline void ixgbevf_rx_checksum(struct ixgbevf_ring *ring,
 361                                        union ixgbe_adv_rx_desc *rx_desc,
 362                                        struct sk_buff *skb)
 363 {
 364         skb_checksum_none_assert(skb);
 365
 366         /* Rx csum disabled */
 367         if (!(ring->netdev->features & NETIF_F_RXCSUM))
 368                 return;
 369
 370         /* if IP and error */
 371         if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) &&
 372             ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) {
 373                 ring->rx_stats.csum_err++;
 374                 return;
 375         }
 376
 377         if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_L4CS))
 378                 return;
 379
 380         if (ixgbevf_test_staterr(rx_desc, IXGBE_RXDADV_ERR_TCPE)) {
 381                 ring->rx_stats.csum_err++;
 382                 return;
 383         }
 384
 385         /* It must be a TCP or UDP packet with a valid checksum */
 386         skb->ip_summed = CHECKSUM_UNNECESSARY;
 387 }
 388
 389 /* ixgbevf_process_skb_fields - Populate skb header fields from Rx descriptor
 390  * @rx_ring: rx descriptor ring packet is being transacted on
 391  * @rx_desc: pointer to the EOP Rx descriptor
 392  * @skb: pointer to current skb being populated
 393  *
 394  * This function checks the ring, descriptor, and packet information in
 395  * order to populate the checksum, VLAN, protocol, and other fields within
 396  * the skb.
 397  */
 398 static void ixgbevf_process_skb_fields(struct ixgbevf_ring *rx_ring,
 399                                        union ixgbe_adv_rx_desc *rx_desc,
 400                                        struct sk_buff *skb)
 401 {
 402         ixgbevf_rx_checksum(rx_ring, rx_desc, skb);
 403
 404         if (ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) {
 405                 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
 406                 unsigned long *active_vlans = netdev_priv(rx_ring->netdev);
 407
 408                 if (test_bit(vid & VLAN_VID_MASK, active_vlans))
 409                         __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
 410         }
 411
 412         skb->protocol = eth_type_trans(skb, rx_ring->netdev);
 413 }
 414
 415 /**
 416  * ixgbevf_is_non_eop - process handling of non-EOP buffers
 417  * @rx_ring: Rx ring being processed
 418  * @rx_desc: Rx descriptor for current buffer
 419  * @skb: current socket buffer containing buffer in progress
 420  *
 421  * This function updates next to clean.  If the buffer is an EOP buffer
 422  * this function exits returning false, otherwise it will place the
 423  * sk_buff in the next buffer to be chained and return true indicating
 424  * that this is in fact a non-EOP buffer.
 425  **/
 426 static bool ixgbevf_is_non_eop(struct ixgbevf_ring *rx_ring,
 427                                union ixgbe_adv_rx_desc *rx_desc)
 428 {
 429         u32 ntc = rx_ring->next_to_clean + 1;
 430
 431         /* fetch, update, and store next to clean */
 432         ntc = (ntc < rx_ring->count) ? ntc : 0;
 433         rx_ring->next_to_clean = ntc;
 434
 435         prefetch(IXGBEVF_RX_DESC(rx_ring, ntc));
 436
 437         if (likely(ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)))
 438                 return false;
 439
 440         return true;
 441 }
 442
 443 static bool ixgbevf_alloc_mapped_page(struct ixgbevf_ring *rx_ring,
 444                                       struct ixgbevf_rx_buffer *bi)
 445 {
 446         struct page *page = bi->page;
 447         dma_addr_t dma = bi->dma;
 448
 449         /* since we are recycling buffers we should seldom need to alloc */
 450         if (likely(page))
 451                 return true;
 452
 453         /* alloc new page for storage */
 454         page = dev_alloc_page();
 455         if (unlikely(!page)) {
 456                 rx_ring->rx_stats.alloc_rx_page_failed++;
 457                 return false;
 458         }
 459
 460         /* map page for use */
 461         dma = dma_map_page(rx_ring->dev, page, 0,
 462                            PAGE_SIZE, DMA_FROM_DEVICE);
 463
 464         /* if mapping failed free memory back to system since
 465          * there isn't much point in holding memory we can't use
 466          */
 467         if (dma_mapping_error(rx_ring->dev, dma)) {
 468                 __free_page(page);
 469
 470                 rx_ring->rx_stats.alloc_rx_buff_failed++;
 471                 return false;
 472         }
 473
 474         bi->dma = dma;
 475         bi->page = page;
 476         bi->page_offset = 0;
 477
 478         return true;
 479 }
 480
 481 /**
 482  * ixgbevf_alloc_rx_buffers - Replace used receive buffers; packet split
 483  * @rx_ring: rx descriptor ring (for a specific queue) to setup buffers on
 484  * @cleaned_count: number of buffers to replace
 485  **/
 486 static void ixgbevf_alloc_rx_buffers(struct ixgbevf_ring *rx_ring,
 487                                      u16 cleaned_count)
 488 {
 489         union ixgbe_adv_rx_desc *rx_desc;
 490         struct ixgbevf_rx_buffer *bi;
 491         unsigned int i = rx_ring->next_to_use;
 492
 493         /* nothing to do or no valid netdev defined */
 494         if (!cleaned_count || !rx_ring->netdev)
 495                 return;
 496
 497         rx_desc = IXGBEVF_RX_DESC(rx_ring, i);
 498         bi = &rx_ring->rx_buffer_info[i];
 499         i -= rx_ring->count;
 500
 501         do {
 502                 if (!ixgbevf_alloc_mapped_page(rx_ring, bi))
 503                         break;
 504
 505                 /* Refresh the desc even if pkt_addr didn't change
 506                  * because each write-back erases this info.
 507                  */
 508                 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
 509
 510                 rx_desc++;
 511                 bi++;
 512                 i++;
 513                 if (unlikely(!i)) {
 514                         rx_desc = IXGBEVF_RX_DESC(rx_ring, 0);
 515                         bi = rx_ring->rx_buffer_info;
 516                         i -= rx_ring->count;
 517                 }
 518
 519                 /* clear the hdr_addr for the next_to_use descriptor */
 520                 rx_desc->read.hdr_addr = 0;
 521
 522                 cleaned_count--;
 523         } while (cleaned_count);
 524
 525         i += rx_ring->count;
 526
 527         if (rx_ring->next_to_use != i) {
 528                 /* record the next descriptor to use */
 529                 rx_ring->next_to_use = i;
 530
 531                 /* update next to alloc since we have filled the ring */
 532                 rx_ring->next_to_alloc = i;
 533
 534                 /* Force memory writes to complete before letting h/w
 535                  * know there are new descriptors to fetch.  (Only
 536                  * applicable for weak-ordered memory model archs,
 537                  * such as IA-64).
 538                  */
 539                 wmb();
 540                 ixgbevf_write_tail(rx_ring, i);
 541         }
 542 }
 543
 544 /* ixgbevf_pull_tail - ixgbevf specific version of skb_pull_tail
 545  * @rx_ring: rx descriptor ring packet is being transacted on
 546  * @skb: pointer to current skb being adjusted
 547  *
 548  * This function is an ixgbevf specific version of __pskb_pull_tail.  The
 549  * main difference between this version and the original function is that
 550  * this function can make several assumptions about the state of things
 551  * that allow for significant optimizations versus the standard function.
 552  * As a result we can do things like drop a frag and maintain an accurate
 553  * truesize for the skb.
 554  */
 555 static void ixgbevf_pull_tail(struct ixgbevf_ring *rx_ring,
 556                               struct sk_buff *skb)
 557 {
 558         struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
 559         unsigned char *va;
 560         unsigned int pull_len;
 561
 562         /* it is valid to use page_address instead of kmap since we are
 563          * working with pages allocated out of the lomem pool per
 564          * alloc_page(GFP_ATOMIC)
 565          */
 566         va = skb_frag_address(frag);
 567
 568         /* we need the header to contain the greater of either ETH_HLEN or
 569          * 60 bytes if the skb->len is less than 60 for skb_pad.
 570          */
 571         pull_len = eth_get_headlen(va, IXGBEVF_RX_HDR_SIZE);
 572
 573         /* align pull length to size of long to optimize memcpy performance */
 574         skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long)));
 575
 576         /* update all of the pointers */
 577         skb_frag_size_sub(frag, pull_len);
 578         frag->page_offset += pull_len;
 579         skb->data_len -= pull_len;
 580         skb->tail += pull_len;
 581 }
 582
 583 /* ixgbevf_cleanup_headers - Correct corrupted or empty headers
 584  * @rx_ring: rx descriptor ring packet is being transacted on
 585  * @rx_desc: pointer to the EOP Rx descriptor
 586  * @skb: pointer to current skb being fixed
 587  *
 588  * Check for corrupted packet headers caused by senders on the local L2
 589  * embedded NIC switch not setting up their Tx Descriptors right.  These
 590  * should be very rare.
 591  *
 592  * Also address the case where we are pulling data in on pages only
 593  * and as such no data is present in the skb header.
 594  *
 595  * In addition if skb is not at least 60 bytes we need to pad it so that
 596  * it is large enough to qualify as a valid Ethernet frame.
 597  *
 598  * Returns true if an error was encountered and skb was freed.
 599  */
 600 static bool ixgbevf_cleanup_headers(struct ixgbevf_ring *rx_ring,
 601                                     union ixgbe_adv_rx_desc *rx_desc,
 602                                     struct sk_buff *skb)
 603 {
 604         /* verify that the packet does not have any known errors */
 605         if (unlikely(ixgbevf_test_staterr(rx_desc,
 606                                           IXGBE_RXDADV_ERR_FRAME_ERR_MASK))) {
 607                 struct net_device *netdev = rx_ring->netdev;
 608
 609                 if (!(netdev->features & NETIF_F_RXALL)) {
 610                         dev_kfree_skb_any(skb);
 611                         return true;
 612                 }
 613         }
 614
 615         /* place header in linear portion of buffer */
 616         if (skb_is_nonlinear(skb))
 617                 ixgbevf_pull_tail(rx_ring, skb);
 618
 619         /* if eth_skb_pad returns an error the skb was freed */
 620         if (eth_skb_pad(skb))
 621                 return true;
 622
 623         return false;
 624 }
 625
 626 /* ixgbevf_reuse_rx_page - page flip buffer and store it back on the ring
 627  * @rx_ring: rx descriptor ring to store buffers on
 628  * @old_buff: donor buffer to have page reused
 629  *
 630  * Synchronizes page for reuse by the adapter
 631  */
 632 static void ixgbevf_reuse_rx_page(struct ixgbevf_ring *rx_ring,
 633                                   struct ixgbevf_rx_buffer *old_buff)
 634 {
 635         struct ixgbevf_rx_buffer *new_buff;
 636         u16 nta = rx_ring->next_to_alloc;
 637
 638         new_buff = &rx_ring->rx_buffer_info[nta];
 639
 640         /* update, and store next to alloc */
 641         nta++;
 642         rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
 643
 644         /* transfer page from old buffer to new buffer */
 645         new_buff->page = old_buff->page;
 646         new_buff->dma = old_buff->dma;
 647         new_buff->page_offset = old_buff->page_offset;
 648
 649         /* sync the buffer for use by the device */
 650         dma_sync_single_range_for_device(rx_ring->dev, new_buff->dma,
 651                                          new_buff->page_offset,
 652                                          IXGBEVF_RX_BUFSZ,
 653                                          DMA_FROM_DEVICE);
 654 }
 655
 656 static inline bool ixgbevf_page_is_reserved(struct page *page)
 657 {
 658         return (page_to_nid(page) != numa_mem_id()) || page->pfmemalloc;
 659 }
 660
 661 /* ixgbevf_add_rx_frag - Add contents of Rx buffer to sk_buff
 662  * @rx_ring: rx descriptor ring to transact packets on
 663  * @rx_buffer: buffer containing page to add
 664  * @rx_desc: descriptor containing length of buffer written by hardware
 665  * @skb: sk_buff to place the data into
 666  *
 667  * This function will add the data contained in rx_buffer->page to the skb.
 668  * This is done either through a direct copy if the data in the buffer is
 669  * less than the skb header size, otherwise it will just attach the page as
 670  * a frag to the skb.
 671  *
 672  * The function will then update the page offset if necessary and return
 673  * true if the buffer can be reused by the adapter.
 674  */
 675 static bool ixgbevf_add_rx_frag(struct ixgbevf_ring *rx_ring,
 676                                 struct ixgbevf_rx_buffer *rx_buffer,
 677                                 union ixgbe_adv_rx_desc *rx_desc,
 678                                 struct sk_buff *skb)
 679 {
 680         struct page *page = rx_buffer->page;
 681         unsigned int size = le16_to_cpu(rx_desc->wb.upper.length);
 682 #if (PAGE_SIZE < 8192)
 683         unsigned int truesize = IXGBEVF_RX_BUFSZ;
 684 #else
 685         unsigned int truesize = ALIGN(size, L1_CACHE_BYTES);
 686 #endif
 687
 688         if ((size <= IXGBEVF_RX_HDR_SIZE) && !skb_is_nonlinear(skb)) {
 689                 unsigned char *va = page_address(page) + rx_buffer->page_offset;
 690
 691                 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));
 692
 693                 /* page is not reserved, we can reuse buffer as is */
 694                 if (likely(!ixgbevf_page_is_reserved(page)))
 695                         return true;
 696
 697                 /* this page cannot be reused so discard it */
 698                 put_page(page);
 699                 return false;
 700         }
 701
 702         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
 703                         rx_buffer->page_offset, size, truesize);
 704
 705         /* avoid re-using remote pages */
 706         if (unlikely(ixgbevf_page_is_reserved(page)))
 707                 return false;
 708
 709 #if (PAGE_SIZE < 8192)
 710         /* if we are only owner of page we can reuse it */
 711         if (unlikely(page_count(page) != 1))
 712                 return false;
 713
 714         /* flip page offset to other buffer */
 715         rx_buffer->page_offset ^= IXGBEVF_RX_BUFSZ;
 716
 717 #else
 718         /* move offset up to the next cache line */
 719         rx_buffer->page_offset += truesize;
 720
 721         if (rx_buffer->page_offset > (PAGE_SIZE - IXGBEVF_RX_BUFSZ))
 722                 return false;
 723
 724 #endif
 725         /* Even if we own the page, we are not allowed to use atomic_set()
 726          * This would break get_page_unless_zero() users.
 727          */
 728         atomic_inc(&page->_count);
 729
 730         return true;
 731 }
 732
 733 static struct sk_buff *ixgbevf_fetch_rx_buffer(struct ixgbevf_ring *rx_ring,
 734                                                union ixgbe_adv_rx_desc *rx_desc,
 735                                                struct sk_buff *skb)
 736 {
 737         struct ixgbevf_rx_buffer *rx_buffer;
 738         struct page *page;
 739
 740         rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
 741         page = rx_buffer->page;
 742         prefetchw(page);
 743
 744         if (likely(!skb)) {
 745                 void *page_addr = page_address(page) +
 746                                   rx_buffer->page_offset;
 747
 748                 /* prefetch first cache line of first page */
 749                 prefetch(page_addr);
 750 #if L1_CACHE_BYTES < 128
 751                 prefetch(page_addr + L1_CACHE_BYTES);
 752 #endif
 753
 754                 /* allocate a skb to store the frags */
 755                 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
 756                                                 IXGBEVF_RX_HDR_SIZE);
 757                 if (unlikely(!skb)) {
 758                         rx_ring->rx_stats.alloc_rx_buff_failed++;
 759                         return NULL;
 760                 }
 761
 762                 /* we will be copying header into skb->data in
 763                  * pskb_may_pull so it is in our interest to prefetch
 764                  * it now to avoid a possible cache miss
 765                  */
 766                 prefetchw(skb->data);
 767         }
 768
 769         /* we are reusing so sync this buffer for CPU use */
 770         dma_sync_single_range_for_cpu(rx_ring->dev,
 771                                       rx_buffer->dma,
 772                                       rx_buffer->page_offset,
 773                                       IXGBEVF_RX_BUFSZ,
 774                                       DMA_FROM_DEVICE);
 775
 776         /* pull page into skb */
 777         if (ixgbevf_add_rx_frag(rx_ring, rx_buffer, rx_desc, skb)) {
 778                 /* hand second half of page back to the ring */
 779                 ixgbevf_reuse_rx_page(rx_ring, rx_buffer);
 780         } else {
 781                 /* we are not reusing the buffer so unmap it */
 782                 dma_unmap_page(rx_ring->dev, rx_buffer->dma,
 783                                PAGE_SIZE, DMA_FROM_DEVICE);
 784         }
 785
 786         /* clear contents of buffer_info */
 787         rx_buffer->dma = 0;
 788         rx_buffer->page = NULL;
 789
 790         return skb;
 791 }
 792
 793 static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter,
 794                                              u32 qmask)
 795 {
 796         struct ixgbe_hw *hw = &adapter->hw;
 797
 798         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, qmask);
 799 }
 800
 801 static int ixgbevf_clean_rx_irq(struct ixgbevf_q_vector *q_vector,
 802                                 struct ixgbevf_ring *rx_ring,
 803                                 int budget)
 804 {
 805         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
 806         u16 cleaned_count = ixgbevf_desc_unused(rx_ring);
 807         struct sk_buff *skb = rx_ring->skb;
 808
 809         while (likely(total_rx_packets < budget)) {
 810                 union ixgbe_adv_rx_desc *rx_desc;
 811
 812                 /* return some buffers to hardware, one at a time is too slow */
 813                 if (cleaned_count >= IXGBEVF_RX_BUFFER_WRITE) {
 814                         ixgbevf_alloc_rx_buffers(rx_ring, cleaned_count);
 815                         cleaned_count = 0;
 816                 }
 817
 818                 rx_desc = IXGBEVF_RX_DESC(rx_ring, rx_ring->next_to_clean);
 819
 820                 if (!ixgbevf_test_staterr(rx_desc, IXGBE_RXD_STAT_DD))
 821                         break;
 822
 823                 /* This memory barrier is needed to keep us from reading
 824                  * any other fields out of the rx_desc until we know the
 825                  * RXD_STAT_DD bit is set
 826                  */
 827                 rmb();
 828
 829                 /* retrieve a buffer from the ring */
 830                 skb = ixgbevf_fetch_rx_buffer(rx_ring, rx_desc, skb);
 831
 832                 /* exit if we failed to retrieve a buffer */
 833                 if (!skb)
 834                         break;
 835
 836                 cleaned_count++;
 837
 838                 /* fetch next buffer in frame if non-eop */
 839                 if (ixgbevf_is_non_eop(rx_ring, rx_desc))
 840                         continue;
 841
 842                 /* verify the packet layout is correct */
 843                 if (ixgbevf_cleanup_headers(rx_ring, rx_desc, skb)) {
 844                         skb = NULL;
 845                         continue;
 846                 }
 847
 848                 /* probably a little skewed due to removing CRC */
 849                 total_rx_bytes += skb->len;
 850
 851                 /* Workaround hardware that can't do proper VEPA multicast
 852                  * source pruning.
 853                  */
 854                 if ((skb->pkt_type == PACKET_BROADCAST ||
 855                     skb->pkt_type == PACKET_MULTICAST) &&
 856                     ether_addr_equal(rx_ring->netdev->dev_addr,
 857                                      eth_hdr(skb)->h_source)) {
 858                         dev_kfree_skb_irq(skb);
 859                         continue;
 860                 }
 861
 862                 /* populate checksum, VLAN, and protocol */
 863                 ixgbevf_process_skb_fields(rx_ring, rx_desc, skb);
 864
 865                 ixgbevf_rx_skb(q_vector, skb);
 866
 867                 /* reset skb pointer */
 868                 skb = NULL;
 869
 870                 /* update budget accounting */
 871                 total_rx_packets++;
 872         }
 873
 874         /* place incomplete frames back on ring for completion */
 875         rx_ring->skb = skb;
 876
 877         u64_stats_update_begin(&rx_ring->syncp);
 878         rx_ring->stats.packets += total_rx_packets;
 879         rx_ring->stats.bytes += total_rx_bytes;
 880         u64_stats_update_end(&rx_ring->syncp);
 881         q_vector->rx.total_packets += total_rx_packets;
 882         q_vector->rx.total_bytes += total_rx_bytes;
 883
 884         return total_rx_packets;
 885 }
 886
 887 /**
 888  * ixgbevf_poll - NAPI polling calback
 889  * @napi: napi struct with our devices info in it
 890  * @budget: amount of work driver is allowed to do this pass, in packets
 891  *
 892  * This function will clean more than one or more rings associated with a
 893  * q_vector.
 894  **/
 895 static int ixgbevf_poll(struct napi_struct *napi, int budget)
 896 {
 897         struct ixgbevf_q_vector *q_vector =
 898                 container_of(napi, struct ixgbevf_q_vector, napi);
 899         struct ixgbevf_adapter *adapter = q_vector->adapter;
 900         struct ixgbevf_ring *ring;
 901         int per_ring_budget;
 902         bool clean_complete = true;
 903
 904         ixgbevf_for_each_ring(ring, q_vector->tx)
 905                 clean_complete &= ixgbevf_clean_tx_irq(q_vector, ring);
 906
 907 #ifdef CONFIG_NET_RX_BUSY_POLL
 908         if (!ixgbevf_qv_lock_napi(q_vector))
 909                 return budget;
 910 #endif
 911
 912         /* attempt to distribute budget to each queue fairly, but don't allow
 913          * the budget to go below 1 because we'll exit polling */
 914         if (q_vector->rx.count > 1)
 915                 per_ring_budget = max(budget/q_vector->rx.count, 1);
 916         else
 917                 per_ring_budget = budget;
 918
 919         ixgbevf_for_each_ring(ring, q_vector->rx)
 920                 clean_complete &= (ixgbevf_clean_rx_irq(q_vector, ring,
 921                                                         per_ring_budget)
 922                                    < per_ring_budget);
 923
 924 #ifdef CONFIG_NET_RX_BUSY_POLL
 925         ixgbevf_qv_unlock_napi(q_vector);
 926 #endif
 927
 928         /* If all work not completed, return budget and keep polling */
 929         if (!clean_complete)
 930                 return budget;
 931         /* all work done, exit the polling mode */
 932         napi_complete(napi);
 933         if (adapter->rx_itr_setting & 1)
 934                 ixgbevf_set_itr(q_vector);
 935         if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
 936             !test_bit(__IXGBEVF_REMOVING, &adapter->state))
 937                 ixgbevf_irq_enable_queues(adapter,
 938                                           1 << q_vector->v_idx);
 939
 940         return 0;
 941 }
 942
 943 /**
 944  * ixgbevf_write_eitr - write VTEITR register in hardware specific way
 945  * @q_vector: structure containing interrupt and ring information
 946  */
 947 void ixgbevf_write_eitr(struct ixgbevf_q_vector *q_vector)
 948 {
 949         struct ixgbevf_adapter *adapter = q_vector->adapter;
 950         struct ixgbe_hw *hw = &adapter->hw;
 951         int v_idx = q_vector->v_idx;
 952         u32 itr_reg = q_vector->itr & IXGBE_MAX_EITR;
 953
 954         /*
 955          * set the WDIS bit to not clear the timer bits and cause an
 956          * immediate assertion of the interrupt
 957          */
 958         itr_reg |= IXGBE_EITR_CNT_WDIS;
 959
 960         IXGBE_WRITE_REG(hw, IXGBE_VTEITR(v_idx), itr_reg);
 961 }
 962
 963 #ifdef CONFIG_NET_RX_BUSY_POLL
 964 /* must be called with local_bh_disable()d */
 965 static int ixgbevf_busy_poll_recv(struct napi_struct *napi)
 966 {
 967         struct ixgbevf_q_vector *q_vector =
 968                         container_of(napi, struct ixgbevf_q_vector, napi);
 969         struct ixgbevf_adapter *adapter = q_vector->adapter;
 970         struct ixgbevf_ring  *ring;
 971         int found = 0;
 972
 973         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
 974                 return LL_FLUSH_FAILED;
 975
 976         if (!ixgbevf_qv_lock_poll(q_vector))
 977                 return LL_FLUSH_BUSY;
 978
 979         ixgbevf_for_each_ring(ring, q_vector->rx) {
 980                 found = ixgbevf_clean_rx_irq(q_vector, ring, 4);
 981 #ifdef BP_EXTENDED_STATS
 982                 if (found)
 983                         ring->stats.cleaned += found;
 984                 else
 985                         ring->stats.misses++;
 986 #endif
 987                 if (found)
 988                         break;
 989         }
 990
 991         ixgbevf_qv_unlock_poll(q_vector);
 992
 993         return found;
 994 }
 995 #endif /* CONFIG_NET_RX_BUSY_POLL */
 996
 997 /**
 998  * ixgbevf_configure_msix - Configure MSI-X hardware
 999  * @adapter: board private structure
1000  *
1001  * ixgbevf_configure_msix sets up the hardware to properly generate MSI-X
1002  * interrupts.
1003  **/
1004 static void ixgbevf_configure_msix(struct ixgbevf_adapter *adapter)
1005 {
1006         struct ixgbevf_q_vector *q_vector;
1007         int q_vectors, v_idx;
1008
1009         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1010         adapter->eims_enable_mask = 0;
1011
1012         /*
1013          * Populate the IVAR table and set the ITR values to the
1014          * corresponding register.
1015          */
1016         for (v_idx = 0; v_idx < q_vectors; v_idx++) {
1017                 struct ixgbevf_ring *ring;
1018                 q_vector = adapter->q_vector[v_idx];
1019
1020                 ixgbevf_for_each_ring(ring, q_vector->rx)
1021                         ixgbevf_set_ivar(adapter, 0, ring->reg_idx, v_idx);
1022
1023                 ixgbevf_for_each_ring(ring, q_vector->tx)
1024                         ixgbevf_set_ivar(adapter, 1, ring->reg_idx, v_idx);
1025
1026                 if (q_vector->tx.ring && !q_vector->rx.ring) {
1027                         /* tx only vector */
1028                         if (adapter->tx_itr_setting == 1)
1029                                 q_vector->itr = IXGBE_10K_ITR;
1030                         else
1031                                 q_vector->itr = adapter->tx_itr_setting;
1032                 } else {
1033                         /* rx or rx/tx vector */
1034                         if (adapter->rx_itr_setting == 1)
1035                                 q_vector->itr = IXGBE_20K_ITR;
1036                         else
1037                                 q_vector->itr = adapter->rx_itr_setting;
1038                 }
1039
1040                 /* add q_vector eims value to global eims_enable_mask */
1041                 adapter->eims_enable_mask |= 1 << v_idx;
1042
1043                 ixgbevf_write_eitr(q_vector);
1044         }
1045
1046         ixgbevf_set_ivar(adapter, -1, 1, v_idx);
1047         /* setup eims_other and add value to global eims_enable_mask */
1048         adapter->eims_other = 1 << v_idx;
1049         adapter->eims_enable_mask |= adapter->eims_other;
1050 }
1051
1052 enum latency_range {
1053         lowest_latency = 0,
1054         low_latency = 1,
1055         bulk_latency = 2,
1056         latency_invalid = 255
1057 };
1058
1059 /**
1060  * ixgbevf_update_itr - update the dynamic ITR value based on statistics
1061  * @q_vector: structure containing interrupt and ring information
1062  * @ring_container: structure containing ring performance data
1063  *
1064  *      Stores a new ITR value based on packets and byte
1065  *      counts during the last interrupt.  The advantage of per interrupt
1066  *      computation is faster updates and more accurate ITR for the current
1067  *      traffic pattern.  Constants in this function were computed
1068  *      based on theoretical maximum wire speed and thresholds were set based
1069  *      on testing data as well as attempting to minimize response time
1070  *      while increasing bulk throughput.
1071  **/
1072 static void ixgbevf_update_itr(struct ixgbevf_q_vector *q_vector,
1073                                struct ixgbevf_ring_container *ring_container)
1074 {
1075         int bytes = ring_container->total_bytes;
1076         int packets = ring_container->total_packets;
1077         u32 timepassed_us;
1078         u64 bytes_perint;
1079         u8 itr_setting = ring_container->itr;
1080
1081         if (packets == 0)
1082                 return;
1083
1084         /* simple throttlerate management
1085          *    0-20MB/s lowest (100000 ints/s)
1086          *   20-100MB/s low   (20000 ints/s)
1087          *  100-1249MB/s bulk (8000 ints/s)
1088          */
1089         /* what was last interrupt timeslice? */
1090         timepassed_us = q_vector->itr >> 2;
1091         bytes_perint = bytes / timepassed_us; /* bytes/usec */
1092
1093         switch (itr_setting) {
1094         case lowest_latency:
1095                 if (bytes_perint > 10)
1096                         itr_setting = low_latency;
1097                 break;
1098         case low_latency:
1099                 if (bytes_perint > 20)
1100                         itr_setting = bulk_latency;
1101                 else if (bytes_perint <= 10)
1102                         itr_setting = lowest_latency;
1103                 break;
1104         case bulk_latency:
1105                 if (bytes_perint <= 20)
1106                         itr_setting = low_latency;
1107                 break;
1108         }
1109
1110         /* clear work counters since we have the values we need */
1111         ring_container->total_bytes = 0;
1112         ring_container->total_packets = 0;
1113
1114         /* write updated itr to ring container */
1115         ring_container->itr = itr_setting;
1116 }
1117
1118 static void ixgbevf_set_itr(struct ixgbevf_q_vector *q_vector)
1119 {
1120         u32 new_itr = q_vector->itr;
1121         u8 current_itr;
1122
1123         ixgbevf_update_itr(q_vector, &q_vector->tx);
1124         ixgbevf_update_itr(q_vector, &q_vector->rx);
1125
1126         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
1127
1128         switch (current_itr) {
1129         /* counts and packets in update_itr are dependent on these numbers */
1130         case lowest_latency:
1131                 new_itr = IXGBE_100K_ITR;
1132                 break;
1133         case low_latency:
1134                 new_itr = IXGBE_20K_ITR;
1135                 break;
1136         case bulk_latency:
1137         default:
1138                 new_itr = IXGBE_8K_ITR;
1139                 break;
1140         }
1141
1142         if (new_itr != q_vector->itr) {
1143                 /* do an exponential smoothing */
1144                 new_itr = (10 * new_itr * q_vector->itr) /
1145                           ((9 * new_itr) + q_vector->itr);
1146
1147                 /* save the algorithm value here */
1148                 q_vector->itr = new_itr;
1149
1150                 ixgbevf_write_eitr(q_vector);
1151         }
1152 }
1153
1154 static irqreturn_t ixgbevf_msix_other(int irq, void *data)
1155 {
1156         struct ixgbevf_adapter *adapter = data;
1157         struct ixgbe_hw *hw = &adapter->hw;
1158
1159         hw->mac.get_link_status = 1;
1160
1161         if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
1162             !test_bit(__IXGBEVF_REMOVING, &adapter->state))
1163                 mod_timer(&adapter->watchdog_timer, jiffies);
1164
1165         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_other);
1166
1167         return IRQ_HANDLED;
1168 }
1169
1170 /**
1171  * ixgbevf_msix_clean_rings - single unshared vector rx clean (all queues)
1172  * @irq: unused
1173  * @data: pointer to our q_vector struct for this interrupt vector
1174  **/
1175 static irqreturn_t ixgbevf_msix_clean_rings(int irq, void *data)
1176 {
1177         struct ixgbevf_q_vector *q_vector = data;
1178
1179         /* EIAM disabled interrupts (on this vector) for us */
1180         if (q_vector->rx.ring || q_vector->tx.ring)
1181                 napi_schedule(&q_vector->napi);
1182
1183         return IRQ_HANDLED;
1184 }
1185
1186 static inline void map_vector_to_rxq(struct ixgbevf_adapter *a, int v_idx,
1187                                      int r_idx)
1188 {
1189         struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
1190
1191         a->rx_ring[r_idx]->next = q_vector->rx.ring;
1192         q_vector->rx.ring = a->rx_ring[r_idx];
1193         q_vector->rx.count++;
1194 }
1195
1196 static inline void map_vector_to_txq(struct ixgbevf_adapter *a, int v_idx,
1197                                      int t_idx)
1198 {
1199         struct ixgbevf_q_vector *q_vector = a->q_vector[v_idx];
1200
1201         a->tx_ring[t_idx]->next = q_vector->tx.ring;
1202         q_vector->tx.ring = a->tx_ring[t_idx];
1203         q_vector->tx.count++;
1204 }
1205
1206 /**
1207  * ixgbevf_map_rings_to_vectors - Maps descriptor rings to vectors
1208  * @adapter: board private structure to initialize
1209  *
1210  * This function maps descriptor rings to the queue-specific vectors
1211  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
1212  * one vector per ring/queue, but on a constrained vector budget, we
1213  * group the rings as "efficiently" as possible.  You would add new
1214  * mapping configurations in here.
1215  **/
1216 static int ixgbevf_map_rings_to_vectors(struct ixgbevf_adapter *adapter)
1217 {
1218         int q_vectors;
1219         int v_start = 0;
1220         int rxr_idx = 0, txr_idx = 0;
1221         int rxr_remaining = adapter->num_rx_queues;
1222         int txr_remaining = adapter->num_tx_queues;
1223         int i, j;
1224         int rqpv, tqpv;
1225         int err = 0;
1226
1227         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1228
1229         /*
1230          * The ideal configuration...
1231          * We have enough vectors to map one per queue.
1232          */
1233         if (q_vectors == adapter->num_rx_queues + adapter->num_tx_queues) {
1234                 for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++)
1235                         map_vector_to_rxq(adapter, v_start, rxr_idx);
1236
1237                 for (; txr_idx < txr_remaining; v_start++, txr_idx++)
1238                         map_vector_to_txq(adapter, v_start, txr_idx);
1239                 goto out;
1240         }
1241
1242         /*
1243          * If we don't have enough vectors for a 1-to-1
1244          * mapping, we'll have to group them so there are
1245          * multiple queues per vector.
1246          */
1247         /* Re-adjusting *qpv takes care of the remainder. */
1248         for (i = v_start; i < q_vectors; i++) {
1249                 rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - i);
1250                 for (j = 0; j < rqpv; j++) {
1251                         map_vector_to_rxq(adapter, i, rxr_idx);
1252                         rxr_idx++;
1253                         rxr_remaining--;
1254                 }
1255         }
1256         for (i = v_start; i < q_vectors; i++) {
1257                 tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - i);
1258                 for (j = 0; j < tqpv; j++) {
1259                         map_vector_to_txq(adapter, i, txr_idx);
1260                         txr_idx++;
1261                         txr_remaining--;
1262                 }
1263         }
1264
1265 out:
1266         return err;
1267 }
1268
1269 /**
1270  * ixgbevf_request_msix_irqs - Initialize MSI-X interrupts
1271  * @adapter: board private structure
1272  *
1273  * ixgbevf_request_msix_irqs allocates MSI-X vectors and requests
1274  * interrupts from the kernel.
1275  **/
1276 static int ixgbevf_request_msix_irqs(struct ixgbevf_adapter *adapter)
1277 {
1278         struct net_device *netdev = adapter->netdev;
1279         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1280         int vector, err;
1281         int ri = 0, ti = 0;
1282
1283         for (vector = 0; vector < q_vectors; vector++) {
1284                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[vector];
1285                 struct msix_entry *entry = &adapter->msix_entries[vector];
1286
1287                 if (q_vector->tx.ring && q_vector->rx.ring) {
1288                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1289                                  "%s-%s-%d", netdev->name, "TxRx", ri++);
1290                         ti++;
1291                 } else if (q_vector->rx.ring) {
1292                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1293                                  "%s-%s-%d", netdev->name, "rx", ri++);
1294                 } else if (q_vector->tx.ring) {
1295                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1296                                  "%s-%s-%d", netdev->name, "tx", ti++);
1297                 } else {
1298                         /* skip this unused q_vector */
1299                         continue;
1300                 }
1301                 err = request_irq(entry->vector, &ixgbevf_msix_clean_rings, 0,
1302                                   q_vector->name, q_vector);
1303                 if (err) {
1304                         hw_dbg(&adapter->hw,
1305                                "request_irq failed for MSIX interrupt "
1306                                "Error: %d\n", err);
1307                         goto free_queue_irqs;
1308                 }
1309         }
1310
1311         err = request_irq(adapter->msix_entries[vector].vector,
1312                           &ixgbevf_msix_other, 0, netdev->name, adapter);
1313         if (err) {
1314                 hw_dbg(&adapter->hw,
1315                        "request_irq for msix_other failed: %d\n", err);
1316                 goto free_queue_irqs;
1317         }
1318
1319         return 0;
1320
1321 free_queue_irqs:
1322         while (vector) {
1323                 vector--;
1324                 free_irq(adapter->msix_entries[vector].vector,
1325                          adapter->q_vector[vector]);
1326         }
1327         /* This failure is non-recoverable - it indicates the system is
1328          * out of MSIX vector resources and the VF driver cannot run
1329          * without them.  Set the number of msix vectors to zero
1330          * indicating that not enough can be allocated.  The error
1331          * will be returned to the user indicating device open failed.
1332          * Any further attempts to force the driver to open will also
1333          * fail.  The only way to recover is to unload the driver and
1334          * reload it again.  If the system has recovered some MSIX
1335          * vectors then it may succeed.
1336          */
1337         adapter->num_msix_vectors = 0;
1338         return err;
1339 }
1340
1341 static inline void ixgbevf_reset_q_vectors(struct ixgbevf_adapter *adapter)
1342 {
1343         int i, q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1344
1345         for (i = 0; i < q_vectors; i++) {
1346                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[i];
1347                 q_vector->rx.ring = NULL;
1348                 q_vector->tx.ring = NULL;
1349                 q_vector->rx.count = 0;
1350                 q_vector->tx.count = 0;
1351         }
1352 }
1353
1354 /**
1355  * ixgbevf_request_irq - initialize interrupts
1356  * @adapter: board private structure
1357  *
1358  * Attempts to configure interrupts using the best available
1359  * capabilities of the hardware and kernel.
1360  **/
1361 static int ixgbevf_request_irq(struct ixgbevf_adapter *adapter)
1362 {
1363         int err = 0;
1364
1365         err = ixgbevf_request_msix_irqs(adapter);
1366
1367         if (err)
1368                 hw_dbg(&adapter->hw,
1369                        "request_irq failed, Error %d\n", err);
1370
1371         return err;
1372 }
1373
1374 static void ixgbevf_free_irq(struct ixgbevf_adapter *adapter)
1375 {
1376         int i, q_vectors;
1377
1378         q_vectors = adapter->num_msix_vectors;
1379         i = q_vectors - 1;
1380
1381         free_irq(adapter->msix_entries[i].vector, adapter);
1382         i--;
1383
1384         for (; i >= 0; i--) {
1385                 /* free only the irqs that were actually requested */
1386                 if (!adapter->q_vector[i]->rx.ring &&
1387                     !adapter->q_vector[i]->tx.ring)
1388                         continue;
1389
1390                 free_irq(adapter->msix_entries[i].vector,
1391                          adapter->q_vector[i]);
1392         }
1393
1394         ixgbevf_reset_q_vectors(adapter);
1395 }
1396
1397 /**
1398  * ixgbevf_irq_disable - Mask off interrupt generation on the NIC
1399  * @adapter: board private structure
1400  **/
1401 static inline void ixgbevf_irq_disable(struct ixgbevf_adapter *adapter)
1402 {
1403         struct ixgbe_hw *hw = &adapter->hw;
1404         int i;
1405
1406         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, 0);
1407         IXGBE_WRITE_REG(hw, IXGBE_VTEIMC, ~0);
1408         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, 0);
1409
1410         IXGBE_WRITE_FLUSH(hw);
1411
1412         for (i = 0; i < adapter->num_msix_vectors; i++)
1413                 synchronize_irq(adapter->msix_entries[i].vector);
1414 }
1415
1416 /**
1417  * ixgbevf_irq_enable - Enable default interrupt generation settings
1418  * @adapter: board private structure
1419  **/
1420 static inline void ixgbevf_irq_enable(struct ixgbevf_adapter *adapter)
1421 {
1422         struct ixgbe_hw *hw = &adapter->hw;
1423
1424         IXGBE_WRITE_REG(hw, IXGBE_VTEIAM, adapter->eims_enable_mask);
1425         IXGBE_WRITE_REG(hw, IXGBE_VTEIAC, adapter->eims_enable_mask);
1426         IXGBE_WRITE_REG(hw, IXGBE_VTEIMS, adapter->eims_enable_mask);
1427 }
1428
1429 /**
1430  * ixgbevf_configure_tx_ring - Configure 82599 VF Tx ring after Reset
1431  * @adapter: board private structure
1432  * @ring: structure containing ring specific data
1433  *
1434  * Configure the Tx descriptor ring after a reset.
1435  **/
1436 static void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter,
1437                                       struct ixgbevf_ring *ring)
1438 {
1439         struct ixgbe_hw *hw = &adapter->hw;
1440         u64 tdba = ring->dma;
1441         int wait_loop = 10;
1442         u32 txdctl = IXGBE_TXDCTL_ENABLE;
1443         u8 reg_idx = ring->reg_idx;
1444
1445         /* disable queue to avoid issues while updating state */
1446         IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), IXGBE_TXDCTL_SWFLSH);
1447         IXGBE_WRITE_FLUSH(hw);
1448
1449         IXGBE_WRITE_REG(hw, IXGBE_VFTDBAL(reg_idx), tdba & DMA_BIT_MASK(32));
1450         IXGBE_WRITE_REG(hw, IXGBE_VFTDBAH(reg_idx), tdba >> 32);
1451         IXGBE_WRITE_REG(hw, IXGBE_VFTDLEN(reg_idx),
1452                         ring->count * sizeof(union ixgbe_adv_tx_desc));
1453
1454         /* disable head writeback */
1455         IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAH(reg_idx), 0);
1456         IXGBE_WRITE_REG(hw, IXGBE_VFTDWBAL(reg_idx), 0);
1457
1458         /* enable relaxed ordering */
1459         IXGBE_WRITE_REG(hw, IXGBE_VFDCA_TXCTRL(reg_idx),
1460                         (IXGBE_DCA_TXCTRL_DESC_RRO_EN |
1461                          IXGBE_DCA_TXCTRL_DATA_RRO_EN));
1462
1463         /* reset head and tail pointers */
1464         IXGBE_WRITE_REG(hw, IXGBE_VFTDH(reg_idx), 0);
1465         IXGBE_WRITE_REG(hw, IXGBE_VFTDT(reg_idx), 0);
1466         ring->tail = adapter->io_addr + IXGBE_VFTDT(reg_idx);
1467
1468         /* reset ntu and ntc to place SW in sync with hardwdare */
1469         ring->next_to_clean = 0;
1470         ring->next_to_use = 0;
1471
1472         /* In order to avoid issues WTHRESH + PTHRESH should always be equal
1473          * to or less than the number of on chip descriptors, which is
1474          * currently 40.
1475          */
1476         txdctl |= (8 << 16);    /* WTHRESH = 8 */
1477
1478         /* Setting PTHRESH to 32 both improves performance */
1479         txdctl |= (1 << 8) |    /* HTHRESH = 1 */
1480                   32;          /* PTHRESH = 32 */
1481
1482         IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx), txdctl);
1483
1484         /* poll to verify queue is enabled */
1485         do {
1486                 usleep_range(1000, 2000);
1487                 txdctl = IXGBE_READ_REG(hw, IXGBE_VFTXDCTL(reg_idx));
1488         }  while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
1489         if (!wait_loop)
1490                 pr_err("Could not enable Tx Queue %d\n", reg_idx);
1491 }
1492
1493 /**
1494  * ixgbevf_configure_tx - Configure 82599 VF Transmit Unit after Reset
1495  * @adapter: board private structure
1496  *
1497  * Configure the Tx unit of the MAC after a reset.
1498  **/
1499 static void ixgbevf_configure_tx(struct ixgbevf_adapter *adapter)
1500 {
1501         u32 i;
1502
1503         /* Setup the HW Tx Head and Tail descriptor pointers */
1504         for (i = 0; i < adapter->num_tx_queues; i++)
1505                 ixgbevf_configure_tx_ring(adapter, adapter->tx_ring[i]);
1506 }
1507
1508 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
1509
1510 static void ixgbevf_configure_srrctl(struct ixgbevf_adapter *adapter, int index)
1511 {
1512         struct ixgbe_hw *hw = &adapter->hw;
1513         u32 srrctl;
1514
1515         srrctl = IXGBE_SRRCTL_DROP_EN;
1516
1517         srrctl |= IXGBEVF_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT;
1518         srrctl |= IXGBEVF_RX_BUFSZ >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
1519         srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
1520
1521         IXGBE_WRITE_REG(hw, IXGBE_VFSRRCTL(index), srrctl);
1522 }
1523
1524 static void ixgbevf_setup_psrtype(struct ixgbevf_adapter *adapter)
1525 {
1526         struct ixgbe_hw *hw = &adapter->hw;
1527
1528         /* PSRTYPE must be initialized in 82599 */
1529         u32 psrtype = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR |
1530                       IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR |
1531                       IXGBE_PSRTYPE_L2HDR;
1532
1533         if (adapter->num_rx_queues > 1)
1534                 psrtype |= 1 << 29;
1535
1536         IXGBE_WRITE_REG(hw, IXGBE_VFPSRTYPE, psrtype);
1537 }
1538
1539 #define IXGBEVF_MAX_RX_DESC_POLL 10
1540 static void ixgbevf_disable_rx_queue(struct ixgbevf_adapter *adapter,
1541                                      struct ixgbevf_ring *ring)
1542 {
1543         struct ixgbe_hw *hw = &adapter->hw;
1544         int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1545         u32 rxdctl;
1546         u8 reg_idx = ring->reg_idx;
1547
1548         if (IXGBE_REMOVED(hw->hw_addr))
1549                 return;
1550         rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1551         rxdctl &= ~IXGBE_RXDCTL_ENABLE;
1552
1553         /* write value back with RXDCTL.ENABLE bit cleared */
1554         IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1555
1556         /* the hardware may take up to 100us to really disable the rx queue */
1557         do {
1558                 udelay(10);
1559                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1560         } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
1561
1562         if (!wait_loop)
1563                 pr_err("RXDCTL.ENABLE queue %d not cleared while polling\n",
1564                        reg_idx);
1565 }
1566
1567 static void ixgbevf_rx_desc_queue_enable(struct ixgbevf_adapter *adapter,
1568                                          struct ixgbevf_ring *ring)
1569 {
1570         struct ixgbe_hw *hw = &adapter->hw;
1571         int wait_loop = IXGBEVF_MAX_RX_DESC_POLL;
1572         u32 rxdctl;
1573         u8 reg_idx = ring->reg_idx;
1574
1575         if (IXGBE_REMOVED(hw->hw_addr))
1576                 return;
1577         do {
1578                 usleep_range(1000, 2000);
1579                 rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1580         } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
1581
1582         if (!wait_loop)
1583                 pr_err("RXDCTL.ENABLE queue %d not set while polling\n",
1584                        reg_idx);
1585 }
1586
1587 static void ixgbevf_configure_rx_ring(struct ixgbevf_adapter *adapter,
1588                                       struct ixgbevf_ring *ring)
1589 {
1590         struct ixgbe_hw *hw = &adapter->hw;
1591         u64 rdba = ring->dma;
1592         u32 rxdctl;
1593         u8 reg_idx = ring->reg_idx;
1594
1595         /* disable queue to avoid issues while updating state */
1596         rxdctl = IXGBE_READ_REG(hw, IXGBE_VFRXDCTL(reg_idx));
1597         ixgbevf_disable_rx_queue(adapter, ring);
1598
1599         IXGBE_WRITE_REG(hw, IXGBE_VFRDBAL(reg_idx), rdba & DMA_BIT_MASK(32));
1600         IXGBE_WRITE_REG(hw, IXGBE_VFRDBAH(reg_idx), rdba >> 32);
1601         IXGBE_WRITE_REG(hw, IXGBE_VFRDLEN(reg_idx),
1602                         ring->count * sizeof(union ixgbe_adv_rx_desc));
1603
1604         /* enable relaxed ordering */
1605         IXGBE_WRITE_REG(hw, IXGBE_VFDCA_RXCTRL(reg_idx),
1606                         IXGBE_DCA_RXCTRL_DESC_RRO_EN);
1607
1608         /* reset head and tail pointers */
1609         IXGBE_WRITE_REG(hw, IXGBE_VFRDH(reg_idx), 0);
1610         IXGBE_WRITE_REG(hw, IXGBE_VFRDT(reg_idx), 0);
1611         ring->tail = adapter->io_addr + IXGBE_VFRDT(reg_idx);
1612
1613         /* reset ntu and ntc to place SW in sync with hardwdare */
1614         ring->next_to_clean = 0;
1615         ring->next_to_use = 0;
1616         ring->next_to_alloc = 0;
1617
1618         ixgbevf_configure_srrctl(adapter, reg_idx);
1619
1620         /* allow any size packet since we can handle overflow */
1621         rxdctl &= ~IXGBE_RXDCTL_RLPML_EN;
1622
1623         rxdctl |= IXGBE_RXDCTL_ENABLE | IXGBE_RXDCTL_VME;
1624         IXGBE_WRITE_REG(hw, IXGBE_VFRXDCTL(reg_idx), rxdctl);
1625
1626         ixgbevf_rx_desc_queue_enable(adapter, ring);
1627         ixgbevf_alloc_rx_buffers(ring, ixgbevf_desc_unused(ring));
1628 }
1629
1630 /**
1631  * ixgbevf_configure_rx - Configure 82599 VF Receive Unit after Reset
1632  * @adapter: board private structure
1633  *
1634  * Configure the Rx unit of the MAC after a reset.
1635  **/
1636 static void ixgbevf_configure_rx(struct ixgbevf_adapter *adapter)
1637 {
1638         int i;
1639         struct ixgbe_hw *hw = &adapter->hw;
1640         struct net_device *netdev = adapter->netdev;
1641
1642         ixgbevf_setup_psrtype(adapter);
1643
1644         /* notify the PF of our intent to use this size of frame */
1645         ixgbevf_rlpml_set_vf(hw, netdev->mtu + ETH_HLEN + ETH_FCS_LEN);
1646
1647         /* Setup the HW Rx Head and Tail Descriptor Pointers and
1648          * the Base and Length of the Rx Descriptor Ring */
1649         for (i = 0; i < adapter->num_rx_queues; i++)
1650                 ixgbevf_configure_rx_ring(adapter, adapter->rx_ring[i]);
1651 }
1652
1653 static int ixgbevf_vlan_rx_add_vid(struct net_device *netdev,
1654                                    __be16 proto, u16 vid)
1655 {
1656         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1657         struct ixgbe_hw *hw = &adapter->hw;
1658         int err;
1659
1660         spin_lock_bh(&adapter->mbx_lock);
1661
1662         /* add VID to filter table */
1663         err = hw->mac.ops.set_vfta(hw, vid, 0, true);
1664
1665         spin_unlock_bh(&adapter->mbx_lock);
1666
1667         /* translate error return types so error makes sense */
1668         if (err == IXGBE_ERR_MBX)
1669                 return -EIO;
1670
1671         if (err == IXGBE_ERR_INVALID_ARGUMENT)
1672                 return -EACCES;
1673
1674         set_bit(vid, adapter->active_vlans);
1675
1676         return err;
1677 }
1678
1679 static int ixgbevf_vlan_rx_kill_vid(struct net_device *netdev,
1680                                     __be16 proto, u16 vid)
1681 {
1682         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1683         struct ixgbe_hw *hw = &adapter->hw;
1684         int err = -EOPNOTSUPP;
1685
1686         spin_lock_bh(&adapter->mbx_lock);
1687
1688         /* remove VID from filter table */
1689         err = hw->mac.ops.set_vfta(hw, vid, 0, false);
1690
1691         spin_unlock_bh(&adapter->mbx_lock);
1692
1693         clear_bit(vid, adapter->active_vlans);
1694
1695         return err;
1696 }
1697
1698 static void ixgbevf_restore_vlan(struct ixgbevf_adapter *adapter)
1699 {
1700         u16 vid;
1701
1702         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1703                 ixgbevf_vlan_rx_add_vid(adapter->netdev,
1704                                         htons(ETH_P_8021Q), vid);
1705 }
1706
1707 static int ixgbevf_write_uc_addr_list(struct net_device *netdev)
1708 {
1709         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1710         struct ixgbe_hw *hw = &adapter->hw;
1711         int count = 0;
1712
1713         if ((netdev_uc_count(netdev)) > 10) {
1714                 pr_err("Too many unicast filters - No Space\n");
1715                 return -ENOSPC;
1716         }
1717
1718         if (!netdev_uc_empty(netdev)) {
1719                 struct netdev_hw_addr *ha;
1720                 netdev_for_each_uc_addr(ha, netdev) {
1721                         hw->mac.ops.set_uc_addr(hw, ++count, ha->addr);
1722                         udelay(200);
1723                 }
1724         } else {
1725                 /*
1726                  * If the list is empty then send message to PF driver to
1727                  * clear all macvlans on this VF.
1728                  */
1729                 hw->mac.ops.set_uc_addr(hw, 0, NULL);
1730         }
1731
1732         return count;
1733 }
1734
1735 /**
1736  * ixgbevf_set_rx_mode - Multicast and unicast set
1737  * @netdev: network interface device structure
1738  *
1739  * The set_rx_method entry point is called whenever the multicast address
1740  * list, unicast address list or the network interface flags are updated.
1741  * This routine is responsible for configuring the hardware for proper
1742  * multicast mode and configuring requested unicast filters.
1743  **/
1744 static void ixgbevf_set_rx_mode(struct net_device *netdev)
1745 {
1746         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
1747         struct ixgbe_hw *hw = &adapter->hw;
1748
1749         spin_lock_bh(&adapter->mbx_lock);
1750
1751         /* reprogram multicast list */
1752         hw->mac.ops.update_mc_addr_list(hw, netdev);
1753
1754         ixgbevf_write_uc_addr_list(netdev);
1755
1756         spin_unlock_bh(&adapter->mbx_lock);
1757 }
1758
1759 static void ixgbevf_napi_enable_all(struct ixgbevf_adapter *adapter)
1760 {
1761         int q_idx;
1762         struct ixgbevf_q_vector *q_vector;
1763         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1764
1765         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1766                 q_vector = adapter->q_vector[q_idx];
1767 #ifdef CONFIG_NET_RX_BUSY_POLL
1768                 ixgbevf_qv_init_lock(adapter->q_vector[q_idx]);
1769 #endif
1770                 napi_enable(&q_vector->napi);
1771         }
1772 }
1773
1774 static void ixgbevf_napi_disable_all(struct ixgbevf_adapter *adapter)
1775 {
1776         int q_idx;
1777         struct ixgbevf_q_vector *q_vector;
1778         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1779
1780         for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1781                 q_vector = adapter->q_vector[q_idx];
1782                 napi_disable(&q_vector->napi);
1783 #ifdef CONFIG_NET_RX_BUSY_POLL
1784                 while (!ixgbevf_qv_disable(adapter->q_vector[q_idx])) {
1785                         pr_info("QV %d locked\n", q_idx);
1786                         usleep_range(1000, 20000);
1787                 }
1788 #endif /* CONFIG_NET_RX_BUSY_POLL */
1789         }
1790 }
1791
1792 static int ixgbevf_configure_dcb(struct ixgbevf_adapter *adapter)
1793 {
1794         struct ixgbe_hw *hw = &adapter->hw;
1795         unsigned int def_q = 0;
1796         unsigned int num_tcs = 0;
1797         unsigned int num_rx_queues = 1;
1798         int err;
1799
1800         spin_lock_bh(&adapter->mbx_lock);
1801
1802         /* fetch queue configuration from the PF */
1803         err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
1804
1805         spin_unlock_bh(&adapter->mbx_lock);
1806
1807         if (err)
1808                 return err;
1809
1810         if (num_tcs > 1) {
1811                 /* update default Tx ring register index */
1812                 adapter->tx_ring[0]->reg_idx = def_q;
1813
1814                 /* we need as many queues as traffic classes */
1815                 num_rx_queues = num_tcs;
1816         }
1817
1818         /* if we have a bad config abort request queue reset */
1819         if (adapter->num_rx_queues != num_rx_queues) {
1820                 /* force mailbox timeout to prevent further messages */
1821                 hw->mbx.timeout = 0;
1822
1823                 /* wait for watchdog to come around and bail us out */
1824                 adapter->flags |= IXGBEVF_FLAG_QUEUE_RESET_REQUESTED;
1825         }
1826
1827         return 0;
1828 }
1829
1830 static void ixgbevf_configure(struct ixgbevf_adapter *adapter)
1831 {
1832         ixgbevf_configure_dcb(adapter);
1833
1834         ixgbevf_set_rx_mode(adapter->netdev);
1835
1836         ixgbevf_restore_vlan(adapter);
1837
1838         ixgbevf_configure_tx(adapter);
1839         ixgbevf_configure_rx(adapter);
1840 }
1841
1842 static void ixgbevf_save_reset_stats(struct ixgbevf_adapter *adapter)
1843 {
1844         /* Only save pre-reset stats if there are some */
1845         if (adapter->stats.vfgprc || adapter->stats.vfgptc) {
1846                 adapter->stats.saved_reset_vfgprc += adapter->stats.vfgprc -
1847                         adapter->stats.base_vfgprc;
1848                 adapter->stats.saved_reset_vfgptc += adapter->stats.vfgptc -
1849                         adapter->stats.base_vfgptc;
1850                 adapter->stats.saved_reset_vfgorc += adapter->stats.vfgorc -
1851                         adapter->stats.base_vfgorc;
1852                 adapter->stats.saved_reset_vfgotc += adapter->stats.vfgotc -
1853                         adapter->stats.base_vfgotc;
1854                 adapter->stats.saved_reset_vfmprc += adapter->stats.vfmprc -
1855                         adapter->stats.base_vfmprc;
1856         }
1857 }
1858
1859 static void ixgbevf_init_last_counter_stats(struct ixgbevf_adapter *adapter)
1860 {
1861         struct ixgbe_hw *hw = &adapter->hw;
1862
1863         adapter->stats.last_vfgprc = IXGBE_READ_REG(hw, IXGBE_VFGPRC);
1864         adapter->stats.last_vfgorc = IXGBE_READ_REG(hw, IXGBE_VFGORC_LSB);
1865         adapter->stats.last_vfgorc |=
1866                 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGORC_MSB))) << 32);
1867         adapter->stats.last_vfgptc = IXGBE_READ_REG(hw, IXGBE_VFGPTC);
1868         adapter->stats.last_vfgotc = IXGBE_READ_REG(hw, IXGBE_VFGOTC_LSB);
1869         adapter->stats.last_vfgotc |=
1870                 (((u64)(IXGBE_READ_REG(hw, IXGBE_VFGOTC_MSB))) << 32);
1871         adapter->stats.last_vfmprc = IXGBE_READ_REG(hw, IXGBE_VFMPRC);
1872
1873         adapter->stats.base_vfgprc = adapter->stats.last_vfgprc;
1874         adapter->stats.base_vfgorc = adapter->stats.last_vfgorc;
1875         adapter->stats.base_vfgptc = adapter->stats.last_vfgptc;
1876         adapter->stats.base_vfgotc = adapter->stats.last_vfgotc;
1877         adapter->stats.base_vfmprc = adapter->stats.last_vfmprc;
1878 }
1879
1880 static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter)
1881 {
1882         struct ixgbe_hw *hw = &adapter->hw;
1883         int api[] = { ixgbe_mbox_api_11,
1884                       ixgbe_mbox_api_10,
1885                       ixgbe_mbox_api_unknown };
1886         int err = 0, idx = 0;
1887
1888         spin_lock_bh(&adapter->mbx_lock);
1889
1890         while (api[idx] != ixgbe_mbox_api_unknown) {
1891                 err = ixgbevf_negotiate_api_version(hw, api[idx]);
1892                 if (!err)
1893                         break;
1894                 idx++;
1895         }
1896
1897         spin_unlock_bh(&adapter->mbx_lock);
1898 }
1899
1900 static void ixgbevf_up_complete(struct ixgbevf_adapter *adapter)
1901 {
1902         struct net_device *netdev = adapter->netdev;
1903         struct ixgbe_hw *hw = &adapter->hw;
1904
1905         ixgbevf_configure_msix(adapter);
1906
1907         spin_lock_bh(&adapter->mbx_lock);
1908
1909         if (is_valid_ether_addr(hw->mac.addr))
1910                 hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
1911         else
1912                 hw->mac.ops.set_rar(hw, 0, hw->mac.perm_addr, 0);
1913
1914         spin_unlock_bh(&adapter->mbx_lock);
1915
1916         smp_mb__before_atomic();
1917         clear_bit(__IXGBEVF_DOWN, &adapter->state);
1918         ixgbevf_napi_enable_all(adapter);
1919
1920         /* enable transmits */
1921         netif_tx_start_all_queues(netdev);
1922
1923         ixgbevf_save_reset_stats(adapter);
1924         ixgbevf_init_last_counter_stats(adapter);
1925
1926         hw->mac.get_link_status = 1;
1927         mod_timer(&adapter->watchdog_timer, jiffies);
1928 }
1929
1930 void ixgbevf_up(struct ixgbevf_adapter *adapter)
1931 {
1932         struct ixgbe_hw *hw = &adapter->hw;
1933
1934         ixgbevf_configure(adapter);
1935
1936         ixgbevf_up_complete(adapter);
1937
1938         /* clear any pending interrupts, may auto mask */
1939         IXGBE_READ_REG(hw, IXGBE_VTEICR);
1940
1941         ixgbevf_irq_enable(adapter);
1942 }
1943
1944 /**
1945  * ixgbevf_clean_rx_ring - Free Rx Buffers per Queue
1946  * @rx_ring: ring to free buffers from
1947  **/
1948 static void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring)
1949 {
1950         struct device *dev = rx_ring->dev;
1951         unsigned long size;
1952         unsigned int i;
1953
1954         /* Free Rx ring sk_buff */
1955         if (rx_ring->skb) {
1956                 dev_kfree_skb(rx_ring->skb);
1957                 rx_ring->skb = NULL;
1958         }
1959
1960         /* ring already cleared, nothing to do */
1961         if (!rx_ring->rx_buffer_info)
1962                 return;
1963
1964         /* Free all the Rx ring pages */
1965         for (i = 0; i < rx_ring->count; i++) {
1966                 struct ixgbevf_rx_buffer *rx_buffer;
1967
1968                 rx_buffer = &rx_ring->rx_buffer_info[i];
1969                 if (rx_buffer->dma)
1970                         dma_unmap_page(dev, rx_buffer->dma,
1971                                        PAGE_SIZE, DMA_FROM_DEVICE);
1972                 rx_buffer->dma = 0;
1973                 if (rx_buffer->page)
1974                         __free_page(rx_buffer->page);
1975                 rx_buffer->page = NULL;
1976         }
1977
1978         size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
1979         memset(rx_ring->rx_buffer_info, 0, size);
1980
1981         /* Zero out the descriptor ring */
1982         memset(rx_ring->desc, 0, rx_ring->size);
1983 }
1984
1985 /**
1986  * ixgbevf_clean_tx_ring - Free Tx Buffers
1987  * @tx_ring: ring to be cleaned
1988  **/
1989 static void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring)
1990 {
1991         struct ixgbevf_tx_buffer *tx_buffer_info;
1992         unsigned long size;
1993         unsigned int i;
1994
1995         if (!tx_ring->tx_buffer_info)
1996                 return;
1997
1998         /* Free all the Tx ring sk_buffs */
1999         for (i = 0; i < tx_ring->count; i++) {
2000                 tx_buffer_info = &tx_ring->tx_buffer_info[i];
2001                 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
2002         }
2003
2004         size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
2005         memset(tx_ring->tx_buffer_info, 0, size);
2006
2007         memset(tx_ring->desc, 0, tx_ring->size);
2008 }
2009
2010 /**
2011  * ixgbevf_clean_all_rx_rings - Free Rx Buffers for all queues
2012  * @adapter: board private structure
2013  **/
2014 static void ixgbevf_clean_all_rx_rings(struct ixgbevf_adapter *adapter)
2015 {
2016         int i;
2017
2018         for (i = 0; i < adapter->num_rx_queues; i++)
2019                 ixgbevf_clean_rx_ring(adapter->rx_ring[i]);
2020 }
2021
2022 /**
2023  * ixgbevf_clean_all_tx_rings - Free Tx Buffers for all queues
2024  * @adapter: board private structure
2025  **/
2026 static void ixgbevf_clean_all_tx_rings(struct ixgbevf_adapter *adapter)
2027 {
2028         int i;
2029
2030         for (i = 0; i < adapter->num_tx_queues; i++)
2031                 ixgbevf_clean_tx_ring(adapter->tx_ring[i]);
2032 }
2033
2034 void ixgbevf_down(struct ixgbevf_adapter *adapter)
2035 {
2036         struct net_device *netdev = adapter->netdev;
2037         struct ixgbe_hw *hw = &adapter->hw;
2038         int i;
2039
2040         /* signal that we are down to the interrupt handler */
2041         if (test_and_set_bit(__IXGBEVF_DOWN, &adapter->state))
2042                 return; /* do nothing if already down */
2043
2044         /* disable all enabled rx queues */
2045         for (i = 0; i < adapter->num_rx_queues; i++)
2046                 ixgbevf_disable_rx_queue(adapter, adapter->rx_ring[i]);
2047
2048         netif_tx_disable(netdev);
2049
2050         msleep(10);
2051
2052         netif_tx_stop_all_queues(netdev);
2053
2054         ixgbevf_irq_disable(adapter);
2055
2056         ixgbevf_napi_disable_all(adapter);
2057
2058         del_timer_sync(&adapter->watchdog_timer);
2059         /* can't call flush scheduled work here because it can deadlock
2060          * if linkwatch_event tries to acquire the rtnl_lock which we are
2061          * holding */
2062         while (adapter->flags & IXGBE_FLAG_IN_WATCHDOG_TASK)
2063                 msleep(1);
2064
2065         /* disable transmits in the hardware now that interrupts are off */
2066         for (i = 0; i < adapter->num_tx_queues; i++) {
2067                 u8 reg_idx = adapter->tx_ring[i]->reg_idx;
2068
2069                 IXGBE_WRITE_REG(hw, IXGBE_VFTXDCTL(reg_idx),
2070                                 IXGBE_TXDCTL_SWFLSH);
2071         }
2072
2073         netif_carrier_off(netdev);
2074
2075         if (!pci_channel_offline(adapter->pdev))
2076                 ixgbevf_reset(adapter);
2077
2078         ixgbevf_clean_all_tx_rings(adapter);
2079         ixgbevf_clean_all_rx_rings(adapter);
2080 }
2081
2082 void ixgbevf_reinit_locked(struct ixgbevf_adapter *adapter)
2083 {
2084         WARN_ON(in_interrupt());
2085
2086         while (test_and_set_bit(__IXGBEVF_RESETTING, &adapter->state))
2087                 msleep(1);
2088
2089         ixgbevf_down(adapter);
2090         ixgbevf_up(adapter);
2091
2092         clear_bit(__IXGBEVF_RESETTING, &adapter->state);
2093 }
2094
2095 void ixgbevf_reset(struct ixgbevf_adapter *adapter)
2096 {
2097         struct ixgbe_hw *hw = &adapter->hw;
2098         struct net_device *netdev = adapter->netdev;
2099
2100         if (hw->mac.ops.reset_hw(hw)) {
2101                 hw_dbg(hw, "PF still resetting\n");
2102         } else {
2103                 hw->mac.ops.init_hw(hw);
2104                 ixgbevf_negotiate_api(adapter);
2105         }
2106
2107         if (is_valid_ether_addr(adapter->hw.mac.addr)) {
2108                 memcpy(netdev->dev_addr, adapter->hw.mac.addr,
2109                        netdev->addr_len);
2110                 memcpy(netdev->perm_addr, adapter->hw.mac.addr,
2111                        netdev->addr_len);
2112         }
2113 }
2114
2115 static int ixgbevf_acquire_msix_vectors(struct ixgbevf_adapter *adapter,
2116                                         int vectors)
2117 {
2118         int vector_threshold;
2119
2120         /* We'll want at least 2 (vector_threshold):
2121          * 1) TxQ[0] + RxQ[0] handler
2122          * 2) Other (Link Status Change, etc.)
2123          */
2124         vector_threshold = MIN_MSIX_COUNT;
2125
2126         /* The more we get, the more we will assign to Tx/Rx Cleanup
2127          * for the separate queues...where Rx Cleanup >= Tx Cleanup.
2128          * Right now, we simply care about how many we'll get; we'll
2129          * set them up later while requesting irq's.
2130          */
2131         vectors = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
2132                                         vector_threshold, vectors);
2133
2134         if (vectors < 0) {
2135                 dev_err(&adapter->pdev->dev,
2136                         "Unable to allocate MSI-X interrupts\n");
2137                 kfree(adapter->msix_entries);
2138                 adapter->msix_entries = NULL;
2139                 return vectors;
2140         }
2141
2142         /* Adjust for only the vectors we'll use, which is minimum
2143          * of max_msix_q_vectors + NON_Q_VECTORS, or the number of
2144          * vectors we were allocated.
2145          */
2146         adapter->num_msix_vectors = vectors;
2147
2148         return 0;
2149 }
2150
2151 /**
2152  * ixgbevf_set_num_queues - Allocate queues for device, feature dependent
2153  * @adapter: board private structure to initialize
2154  *
2155  * This is the top level queue allocation routine.  The order here is very
2156  * important, starting with the "most" number of features turned on at once,
2157  * and ending with the smallest set of features.  This way large combinations
2158  * can be allocated if they're turned on, and smaller combinations are the
2159  * fallthrough conditions.
2160  *
2161  **/
2162 static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter)
2163 {
2164         struct ixgbe_hw *hw = &adapter->hw;
2165         unsigned int def_q = 0;
2166         unsigned int num_tcs = 0;
2167         int err;
2168
2169         /* Start with base case */
2170         adapter->num_rx_queues = 1;
2171         adapter->num_tx_queues = 1;
2172
2173         spin_lock_bh(&adapter->mbx_lock);
2174
2175         /* fetch queue configuration from the PF */
2176         err = ixgbevf_get_queues(hw, &num_tcs, &def_q);
2177
2178         spin_unlock_bh(&adapter->mbx_lock);
2179
2180         if (err)
2181                 return;
2182
2183         /* we need as many queues as traffic classes */
2184         if (num_tcs > 1)
2185                 adapter->num_rx_queues = num_tcs;
2186 }
2187
2188 /**
2189  * ixgbevf_alloc_queues - Allocate memory for all rings
2190  * @adapter: board private structure to initialize
2191  *
2192  * We allocate one ring per queue at run-time since we don't know the
2193  * number of queues at compile-time.  The polling_netdev array is
2194  * intended for Multiqueue, but should work fine with a single queue.
2195  **/
2196 static int ixgbevf_alloc_queues(struct ixgbevf_adapter *adapter)
2197 {
2198         struct ixgbevf_ring *ring;
2199         int rx = 0, tx = 0;
2200
2201         for (; tx < adapter->num_tx_queues; tx++) {
2202                 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
2203                 if (!ring)
2204                         goto err_allocation;
2205
2206                 ring->dev = &adapter->pdev->dev;
2207                 ring->netdev = adapter->netdev;
2208                 ring->count = adapter->tx_ring_count;
2209                 ring->queue_index = tx;
2210                 ring->reg_idx = tx;
2211
2212                 adapter->tx_ring[tx] = ring;
2213         }
2214
2215         for (; rx < adapter->num_rx_queues; rx++) {
2216                 ring = kzalloc(sizeof(*ring), GFP_KERNEL);
2217                 if (!ring)
2218                         goto err_allocation;
2219
2220                 ring->dev = &adapter->pdev->dev;
2221                 ring->netdev = adapter->netdev;
2222
2223                 ring->count = adapter->rx_ring_count;
2224                 ring->queue_index = rx;
2225                 ring->reg_idx = rx;
2226
2227                 adapter->rx_ring[rx] = ring;
2228         }
2229
2230         return 0;
2231
2232 err_allocation:
2233         while (tx) {
2234                 kfree(adapter->tx_ring[--tx]);
2235                 adapter->tx_ring[tx] = NULL;
2236         }
2237
2238         while (rx) {
2239                 kfree(adapter->rx_ring[--rx]);
2240                 adapter->rx_ring[rx] = NULL;
2241         }
2242         return -ENOMEM;
2243 }
2244
2245 /**
2246  * ixgbevf_set_interrupt_capability - set MSI-X or FAIL if not supported
2247  * @adapter: board private structure to initialize
2248  *
2249  * Attempt to configure the interrupts using the best available
2250  * capabilities of the hardware and the kernel.
2251  **/
2252 static int ixgbevf_set_interrupt_capability(struct ixgbevf_adapter *adapter)
2253 {
2254         struct net_device *netdev = adapter->netdev;
2255         int err = 0;
2256         int vector, v_budget;
2257
2258         /*
2259          * It's easy to be greedy for MSI-X vectors, but it really
2260          * doesn't do us much good if we have a lot more vectors
2261          * than CPU's.  So let's be conservative and only ask for
2262          * (roughly) the same number of vectors as there are CPU's.
2263          * The default is to use pairs of vectors.
2264          */
2265         v_budget = max(adapter->num_rx_queues, adapter->num_tx_queues);
2266         v_budget = min_t(int, v_budget, num_online_cpus());
2267         v_budget += NON_Q_VECTORS;
2268
2269         /* A failure in MSI-X entry allocation isn't fatal, but it does
2270          * mean we disable MSI-X capabilities of the adapter. */
2271         adapter->msix_entries = kcalloc(v_budget,
2272                                         sizeof(struct msix_entry), GFP_KERNEL);
2273         if (!adapter->msix_entries) {
2274                 err = -ENOMEM;
2275                 goto out;
2276         }
2277
2278         for (vector = 0; vector < v_budget; vector++)
2279                 adapter->msix_entries[vector].entry = vector;
2280
2281         err = ixgbevf_acquire_msix_vectors(adapter, v_budget);
2282         if (err)
2283                 goto out;
2284
2285         err = netif_set_real_num_tx_queues(netdev, adapter->num_tx_queues);
2286         if (err)
2287                 goto out;
2288
2289         err = netif_set_real_num_rx_queues(netdev, adapter->num_rx_queues);
2290
2291 out:
2292         return err;
2293 }
2294
2295 /**
2296  * ixgbevf_alloc_q_vectors - Allocate memory for interrupt vectors
2297  * @adapter: board private structure to initialize
2298  *
2299  * We allocate one q_vector per queue interrupt.  If allocation fails we
2300  * return -ENOMEM.
2301  **/
2302 static int ixgbevf_alloc_q_vectors(struct ixgbevf_adapter *adapter)
2303 {
2304         int q_idx, num_q_vectors;
2305         struct ixgbevf_q_vector *q_vector;
2306
2307         num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2308
2309         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2310                 q_vector = kzalloc(sizeof(struct ixgbevf_q_vector), GFP_KERNEL);
2311                 if (!q_vector)
2312                         goto err_out;
2313                 q_vector->adapter = adapter;
2314                 q_vector->v_idx = q_idx;
2315                 netif_napi_add(adapter->netdev, &q_vector->napi,
2316                                ixgbevf_poll, 64);
2317 #ifdef CONFIG_NET_RX_BUSY_POLL
2318                 napi_hash_add(&q_vector->napi);
2319 #endif
2320                 adapter->q_vector[q_idx] = q_vector;
2321         }
2322
2323         return 0;
2324
2325 err_out:
2326         while (q_idx) {
2327                 q_idx--;
2328                 q_vector = adapter->q_vector[q_idx];
2329 #ifdef CONFIG_NET_RX_BUSY_POLL
2330                 napi_hash_del(&q_vector->napi);
2331 #endif
2332                 netif_napi_del(&q_vector->napi);
2333                 kfree(q_vector);
2334                 adapter->q_vector[q_idx] = NULL;
2335         }
2336         return -ENOMEM;
2337 }
2338
2339 /**
2340  * ixgbevf_free_q_vectors - Free memory allocated for interrupt vectors
2341  * @adapter: board private structure to initialize
2342  *
2343  * This function frees the memory allocated to the q_vectors.  In addition if
2344  * NAPI is enabled it will delete any references to the NAPI struct prior
2345  * to freeing the q_vector.
2346  **/
2347 static void ixgbevf_free_q_vectors(struct ixgbevf_adapter *adapter)
2348 {
2349         int q_idx, num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2350
2351         for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
2352                 struct ixgbevf_q_vector *q_vector = adapter->q_vector[q_idx];
2353
2354                 adapter->q_vector[q_idx] = NULL;
2355 #ifdef CONFIG_NET_RX_BUSY_POLL
2356                 napi_hash_del(&q_vector->napi);
2357 #endif
2358                 netif_napi_del(&q_vector->napi);
2359                 kfree(q_vector);
2360         }
2361 }
2362
2363 /**
2364  * ixgbevf_reset_interrupt_capability - Reset MSIX setup
2365  * @adapter: board private structure
2366  *
2367  **/
2368 static void ixgbevf_reset_interrupt_capability(struct ixgbevf_adapter *adapter)
2369 {
2370         pci_disable_msix(adapter->pdev);
2371         kfree(adapter->msix_entries);
2372         adapter->msix_entries = NULL;
2373 }
2374
2375 /**
2376  * ixgbevf_init_interrupt_scheme - Determine if MSIX is supported and init
2377  * @adapter: board private structure to initialize
2378  *
2379  **/
2380 static int ixgbevf_init_interrupt_scheme(struct ixgbevf_adapter *adapter)
2381 {
2382         int err;
2383
2384         /* Number of supported queues */
2385         ixgbevf_set_num_queues(adapter);
2386
2387         err = ixgbevf_set_interrupt_capability(adapter);
2388         if (err) {
2389                 hw_dbg(&adapter->hw,
2390                        "Unable to setup interrupt capabilities\n");
2391                 goto err_set_interrupt;
2392         }
2393
2394         err = ixgbevf_alloc_q_vectors(adapter);
2395         if (err) {
2396                 hw_dbg(&adapter->hw, "Unable to allocate memory for queue "
2397                        "vectors\n");
2398                 goto err_alloc_q_vectors;
2399         }
2400
2401         err = ixgbevf_alloc_queues(adapter);
2402         if (err) {
2403                 pr_err("Unable to allocate memory for queues\n");
2404                 goto err_alloc_queues;
2405         }
2406
2407         hw_dbg(&adapter->hw, "Multiqueue %s: Rx Queue count = %u, "
2408                "Tx Queue count = %u\n",
2409                (adapter->num_rx_queues > 1) ? "Enabled" :
2410                "Disabled", adapter->num_rx_queues, adapter->num_tx_queues);
2411
2412         set_bit(__IXGBEVF_DOWN, &adapter->state);
2413
2414         return 0;
2415 err_alloc_queues:
2416         ixgbevf_free_q_vectors(adapter);
2417 err_alloc_q_vectors:
2418         ixgbevf_reset_interrupt_capability(adapter);
2419 err_set_interrupt:
2420         return err;
2421 }
2422
2423 /**
2424  * ixgbevf_clear_interrupt_scheme - Clear the current interrupt scheme settings
2425  * @adapter: board private structure to clear interrupt scheme on
2426  *
2427  * We go through and clear interrupt specific resources and reset the structure
2428  * to pre-load conditions
2429  **/
2430 static void ixgbevf_clear_interrupt_scheme(struct ixgbevf_adapter *adapter)
2431 {
2432         int i;
2433
2434         for (i = 0; i < adapter->num_tx_queues; i++) {
2435                 kfree(adapter->tx_ring[i]);
2436                 adapter->tx_ring[i] = NULL;
2437         }
2438         for (i = 0; i < adapter->num_rx_queues; i++) {
2439                 kfree(adapter->rx_ring[i]);
2440                 adapter->rx_ring[i] = NULL;
2441         }
2442
2443         adapter->num_tx_queues = 0;
2444         adapter->num_rx_queues = 0;
2445
2446         ixgbevf_free_q_vectors(adapter);
2447         ixgbevf_reset_interrupt_capability(adapter);
2448 }
2449
2450 /**
2451  * ixgbevf_sw_init - Initialize general software structures
2452  * (struct ixgbevf_adapter)
2453  * @adapter: board private structure to initialize
2454  *
2455  * ixgbevf_sw_init initializes the Adapter private data structure.
2456  * Fields are initialized based on PCI device information and
2457  * OS network device settings (MTU size).
2458  **/
2459 static int ixgbevf_sw_init(struct ixgbevf_adapter *adapter)
2460 {
2461         struct ixgbe_hw *hw = &adapter->hw;
2462         struct pci_dev *pdev = adapter->pdev;
2463         struct net_device *netdev = adapter->netdev;
2464         int err;
2465
2466         /* PCI config space info */
2467
2468         hw->vendor_id = pdev->vendor;
2469         hw->device_id = pdev->device;
2470         hw->revision_id = pdev->revision;
2471         hw->subsystem_vendor_id = pdev->subsystem_vendor;
2472         hw->subsystem_device_id = pdev->subsystem_device;
2473
2474         hw->mbx.ops.init_params(hw);
2475
2476         /* assume legacy case in which PF would only give VF 2 queues */
2477         hw->mac.max_tx_queues = 2;
2478         hw->mac.max_rx_queues = 2;
2479
2480         /* lock to protect mailbox accesses */
2481         spin_lock_init(&adapter->mbx_lock);
2482
2483         err = hw->mac.ops.reset_hw(hw);
2484         if (err) {
2485                 dev_info(&pdev->dev,
2486                          "PF still in reset state.  Is the PF interface up?\n");
2487         } else {
2488                 err = hw->mac.ops.init_hw(hw);
2489                 if (err) {
2490                         pr_err("init_shared_code failed: %d\n", err);
2491                         goto out;
2492                 }
2493                 ixgbevf_negotiate_api(adapter);
2494                 err = hw->mac.ops.get_mac_addr(hw, hw->mac.addr);
2495                 if (err)
2496                         dev_info(&pdev->dev, "Error reading MAC address\n");
2497                 else if (is_zero_ether_addr(adapter->hw.mac.addr))
2498                         dev_info(&pdev->dev,
2499                                  "MAC address not assigned by administrator.\n");
2500                 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2501         }
2502
2503         if (!is_valid_ether_addr(netdev->dev_addr)) {
2504                 dev_info(&pdev->dev, "Assigning random MAC address\n");
2505                 eth_hw_addr_random(netdev);
2506                 memcpy(hw->mac.addr, netdev->dev_addr, netdev->addr_len);
2507         }
2508
2509         /* Enable dynamic interrupt throttling rates */
2510         adapter->rx_itr_setting = 1;
2511         adapter->tx_itr_setting = 1;
2512
2513         /* set default ring sizes */
2514         adapter->tx_ring_count = IXGBEVF_DEFAULT_TXD;
2515         adapter->rx_ring_count = IXGBEVF_DEFAULT_RXD;
2516
2517         set_bit(__IXGBEVF_DOWN, &adapter->state);
2518         return 0;
2519
2520 out:
2521         return err;
2522 }
2523
2524 #define UPDATE_VF_COUNTER_32bit(reg, last_counter, counter)     \
2525         {                                                       \
2526                 u32 current_counter = IXGBE_READ_REG(hw, reg);  \
2527                 if (current_counter < last_counter)             \
2528                         counter += 0x100000000LL;               \
2529                 last_counter = current_counter;                 \
2530                 counter &= 0xFFFFFFFF00000000LL;                \
2531                 counter |= current_counter;                     \
2532         }
2533
2534 #define UPDATE_VF_COUNTER_36bit(reg_lsb, reg_msb, last_counter, counter) \
2535         {                                                                \
2536                 u64 current_counter_lsb = IXGBE_READ_REG(hw, reg_lsb);   \
2537                 u64 current_counter_msb = IXGBE_READ_REG(hw, reg_msb);   \
2538                 u64 current_counter = (current_counter_msb << 32) |      \
2539                         current_counter_lsb;                             \
2540                 if (current_counter < last_counter)                      \
2541                         counter += 0x1000000000LL;                       \
2542                 last_counter = current_counter;                          \
2543                 counter &= 0xFFFFFFF000000000LL;                         \
2544                 counter |= current_counter;                              \
2545         }
2546 /**
2547  * ixgbevf_update_stats - Update the board statistics counters.
2548  * @adapter: board private structure
2549  **/
2550 void ixgbevf_update_stats(struct ixgbevf_adapter *adapter)
2551 {
2552         struct ixgbe_hw *hw = &adapter->hw;
2553         int i;
2554
2555         if (!adapter->link_up)
2556                 return;
2557
2558         UPDATE_VF_COUNTER_32bit(IXGBE_VFGPRC, adapter->stats.last_vfgprc,
2559                                 adapter->stats.vfgprc);
2560         UPDATE_VF_COUNTER_32bit(IXGBE_VFGPTC, adapter->stats.last_vfgptc,
2561                                 adapter->stats.vfgptc);
2562         UPDATE_VF_COUNTER_36bit(IXGBE_VFGORC_LSB, IXGBE_VFGORC_MSB,
2563                                 adapter->stats.last_vfgorc,
2564                                 adapter->stats.vfgorc);
2565         UPDATE_VF_COUNTER_36bit(IXGBE_VFGOTC_LSB, IXGBE_VFGOTC_MSB,
2566                                 adapter->stats.last_vfgotc,
2567                                 adapter->stats.vfgotc);
2568         UPDATE_VF_COUNTER_32bit(IXGBE_VFMPRC, adapter->stats.last_vfmprc,
2569                                 adapter->stats.vfmprc);
2570
2571         for (i = 0;  i  < adapter->num_rx_queues;  i++) {
2572                 adapter->hw_csum_rx_error +=
2573                         adapter->rx_ring[i]->hw_csum_rx_error;
2574                 adapter->rx_ring[i]->hw_csum_rx_error = 0;
2575         }
2576 }
2577
2578 /**
2579  * ixgbevf_watchdog - Timer Call-back
2580  * @data: pointer to adapter cast into an unsigned long
2581  **/
2582 static void ixgbevf_watchdog(unsigned long data)
2583 {
2584         struct ixgbevf_adapter *adapter = (struct ixgbevf_adapter *)data;
2585         struct ixgbe_hw *hw = &adapter->hw;
2586         u32 eics = 0;
2587         int i;
2588
2589         /*
2590          * Do the watchdog outside of interrupt context due to the lovely
2591          * delays that some of the newer hardware requires
2592          */
2593
2594         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
2595                 goto watchdog_short_circuit;
2596
2597         /* get one bit for every active tx/rx interrupt vector */
2598         for (i = 0; i < adapter->num_msix_vectors - NON_Q_VECTORS; i++) {
2599                 struct ixgbevf_q_vector *qv = adapter->q_vector[i];
2600                 if (qv->rx.ring || qv->tx.ring)
2601                         eics |= 1 << i;
2602         }
2603
2604         IXGBE_WRITE_REG(hw, IXGBE_VTEICS, eics);
2605
2606 watchdog_short_circuit:
2607         schedule_work(&adapter->watchdog_task);
2608 }
2609
2610 /**
2611  * ixgbevf_tx_timeout - Respond to a Tx Hang
2612  * @netdev: network interface device structure
2613  **/
2614 static void ixgbevf_tx_timeout(struct net_device *netdev)
2615 {
2616         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2617
2618         /* Do the reset outside of interrupt context */
2619         schedule_work(&adapter->reset_task);
2620 }
2621
2622 static void ixgbevf_reset_task(struct work_struct *work)
2623 {
2624         struct ixgbevf_adapter *adapter;
2625         adapter = container_of(work, struct ixgbevf_adapter, reset_task);
2626
2627         /* If we're already down or resetting, just bail */
2628         if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
2629             test_bit(__IXGBEVF_REMOVING, &adapter->state) ||
2630             test_bit(__IXGBEVF_RESETTING, &adapter->state))
2631                 return;
2632
2633         adapter->tx_timeout_count++;
2634
2635         ixgbevf_reinit_locked(adapter);
2636 }
2637
2638 /**
2639  * ixgbevf_watchdog_task - worker thread to bring link up
2640  * @work: pointer to work_struct containing our data
2641  **/
2642 static void ixgbevf_watchdog_task(struct work_struct *work)
2643 {
2644         struct ixgbevf_adapter *adapter = container_of(work,
2645                                                        struct ixgbevf_adapter,
2646                                                        watchdog_task);
2647         struct net_device *netdev = adapter->netdev;
2648         struct ixgbe_hw *hw = &adapter->hw;
2649         u32 link_speed = adapter->link_speed;
2650         bool link_up = adapter->link_up;
2651         s32 need_reset;
2652
2653         if (IXGBE_REMOVED(hw->hw_addr)) {
2654                 if (!test_bit(__IXGBEVF_DOWN, &adapter->state)) {
2655                         rtnl_lock();
2656                         ixgbevf_down(adapter);
2657                         rtnl_unlock();
2658                 }
2659                 return;
2660         }
2661         ixgbevf_queue_reset_subtask(adapter);
2662
2663         adapter->flags |= IXGBE_FLAG_IN_WATCHDOG_TASK;
2664
2665         /*
2666          * Always check the link on the watchdog because we have
2667          * no LSC interrupt
2668          */
2669         spin_lock_bh(&adapter->mbx_lock);
2670
2671         need_reset = hw->mac.ops.check_link(hw, &link_speed, &link_up, false);
2672
2673         spin_unlock_bh(&adapter->mbx_lock);
2674
2675         if (need_reset) {
2676                 adapter->link_up = link_up;
2677                 adapter->link_speed = link_speed;
2678                 netif_carrier_off(netdev);
2679                 netif_tx_stop_all_queues(netdev);
2680                 schedule_work(&adapter->reset_task);
2681                 goto pf_has_reset;
2682         }
2683         adapter->link_up = link_up;
2684         adapter->link_speed = link_speed;
2685
2686         if (link_up) {
2687                 if (!netif_carrier_ok(netdev)) {
2688                         char *link_speed_string;
2689                         switch (link_speed) {
2690                         case IXGBE_LINK_SPEED_10GB_FULL:
2691                                 link_speed_string = "10 Gbps";
2692                                 break;
2693                         case IXGBE_LINK_SPEED_1GB_FULL:
2694                                 link_speed_string = "1 Gbps";
2695                                 break;
2696                         case IXGBE_LINK_SPEED_100_FULL:
2697                                 link_speed_string = "100 Mbps";
2698                                 break;
2699                         default:
2700                                 link_speed_string = "unknown speed";
2701                                 break;
2702                         }
2703                         dev_info(&adapter->pdev->dev,
2704                                 "NIC Link is Up, %s\n", link_speed_string);
2705                         netif_carrier_on(netdev);
2706                         netif_tx_wake_all_queues(netdev);
2707                 }
2708         } else {
2709                 adapter->link_up = false;
2710                 adapter->link_speed = 0;
2711                 if (netif_carrier_ok(netdev)) {
2712                         dev_info(&adapter->pdev->dev, "NIC Link is Down\n");
2713                         netif_carrier_off(netdev);
2714                         netif_tx_stop_all_queues(netdev);
2715                 }
2716         }
2717
2718         ixgbevf_update_stats(adapter);
2719
2720 pf_has_reset:
2721         /* Reset the timer */
2722         if (!test_bit(__IXGBEVF_DOWN, &adapter->state) &&
2723             !test_bit(__IXGBEVF_REMOVING, &adapter->state))
2724                 mod_timer(&adapter->watchdog_timer,
2725                           round_jiffies(jiffies + (2 * HZ)));
2726
2727         adapter->flags &= ~IXGBE_FLAG_IN_WATCHDOG_TASK;
2728 }
2729
2730 /**
2731  * ixgbevf_free_tx_resources - Free Tx Resources per Queue
2732  * @tx_ring: Tx descriptor ring for a specific queue
2733  *
2734  * Free all transmit software resources
2735  **/
2736 void ixgbevf_free_tx_resources(struct ixgbevf_ring *tx_ring)
2737 {
2738         ixgbevf_clean_tx_ring(tx_ring);
2739
2740         vfree(tx_ring->tx_buffer_info);
2741         tx_ring->tx_buffer_info = NULL;
2742
2743         /* if not set, then don't free */
2744         if (!tx_ring->desc)
2745                 return;
2746
2747         dma_free_coherent(tx_ring->dev, tx_ring->size, tx_ring->desc,
2748                           tx_ring->dma);
2749
2750         tx_ring->desc = NULL;
2751 }
2752
2753 /**
2754  * ixgbevf_free_all_tx_resources - Free Tx Resources for All Queues
2755  * @adapter: board private structure
2756  *
2757  * Free all transmit software resources
2758  **/
2759 static void ixgbevf_free_all_tx_resources(struct ixgbevf_adapter *adapter)
2760 {
2761         int i;
2762
2763         for (i = 0; i < adapter->num_tx_queues; i++)
2764                 if (adapter->tx_ring[i]->desc)
2765                         ixgbevf_free_tx_resources(adapter->tx_ring[i]);
2766 }
2767
2768 /**
2769  * ixgbevf_setup_tx_resources - allocate Tx resources (Descriptors)
2770  * @tx_ring:    tx descriptor ring (for a specific queue) to setup
2771  *
2772  * Return 0 on success, negative on failure
2773  **/
2774 int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring)
2775 {
2776         int size;
2777
2778         size = sizeof(struct ixgbevf_tx_buffer) * tx_ring->count;
2779         tx_ring->tx_buffer_info = vzalloc(size);
2780         if (!tx_ring->tx_buffer_info)
2781                 goto err;
2782
2783         /* round up to nearest 4K */
2784         tx_ring->size = tx_ring->count * sizeof(union ixgbe_adv_tx_desc);
2785         tx_ring->size = ALIGN(tx_ring->size, 4096);
2786
2787         tx_ring->desc = dma_alloc_coherent(tx_ring->dev, tx_ring->size,
2788                                            &tx_ring->dma, GFP_KERNEL);
2789         if (!tx_ring->desc)
2790                 goto err;
2791
2792         return 0;
2793
2794 err:
2795         vfree(tx_ring->tx_buffer_info);
2796         tx_ring->tx_buffer_info = NULL;
2797         hw_dbg(&adapter->hw, "Unable to allocate memory for the transmit "
2798                "descriptor ring\n");
2799         return -ENOMEM;
2800 }
2801
2802 /**
2803  * ixgbevf_setup_all_tx_resources - allocate all queues Tx resources
2804  * @adapter: board private structure
2805  *
2806  * If this function returns with an error, then it's possible one or
2807  * more of the rings is populated (while the rest are not).  It is the
2808  * callers duty to clean those orphaned rings.
2809  *
2810  * Return 0 on success, negative on failure
2811  **/
2812 static int ixgbevf_setup_all_tx_resources(struct ixgbevf_adapter *adapter)
2813 {
2814         int i, err = 0;
2815
2816         for (i = 0; i < adapter->num_tx_queues; i++) {
2817                 err = ixgbevf_setup_tx_resources(adapter->tx_ring[i]);
2818                 if (!err)
2819                         continue;
2820                 hw_dbg(&adapter->hw,
2821                        "Allocation for Tx Queue %u failed\n", i);
2822                 break;
2823         }
2824
2825         return err;
2826 }
2827
2828 /**
2829  * ixgbevf_setup_rx_resources - allocate Rx resources (Descriptors)
2830  * @rx_ring:    rx descriptor ring (for a specific queue) to setup
2831  *
2832  * Returns 0 on success, negative on failure
2833  **/
2834 int ixgbevf_setup_rx_resources(struct ixgbevf_ring *rx_ring)
2835 {
2836         int size;
2837
2838         size = sizeof(struct ixgbevf_rx_buffer) * rx_ring->count;
2839         rx_ring->rx_buffer_info = vzalloc(size);
2840         if (!rx_ring->rx_buffer_info)
2841                 goto err;
2842
2843         /* Round up to nearest 4K */
2844         rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc);
2845         rx_ring->size = ALIGN(rx_ring->size, 4096);
2846
2847         rx_ring->desc = dma_alloc_coherent(rx_ring->dev, rx_ring->size,
2848                                            &rx_ring->dma, GFP_KERNEL);
2849
2850         if (!rx_ring->desc)
2851                 goto err;
2852
2853         return 0;
2854 err:
2855         vfree(rx_ring->rx_buffer_info);
2856         rx_ring->rx_buffer_info = NULL;
2857         dev_err(rx_ring->dev, "Unable to allocate memory for the Rx descriptor ring\n");
2858         return -ENOMEM;
2859 }
2860
2861 /**
2862  * ixgbevf_setup_all_rx_resources - allocate all queues Rx resources
2863  * @adapter: board private structure
2864  *
2865  * If this function returns with an error, then it's possible one or
2866  * more of the rings is populated (while the rest are not).  It is the
2867  * callers duty to clean those orphaned rings.
2868  *
2869  * Return 0 on success, negative on failure
2870  **/
2871 static int ixgbevf_setup_all_rx_resources(struct ixgbevf_adapter *adapter)
2872 {
2873         int i, err = 0;
2874
2875         for (i = 0; i < adapter->num_rx_queues; i++) {
2876                 err = ixgbevf_setup_rx_resources(adapter->rx_ring[i]);
2877                 if (!err)
2878                         continue;
2879                 hw_dbg(&adapter->hw,
2880                        "Allocation for Rx Queue %u failed\n", i);
2881                 break;
2882         }
2883         return err;
2884 }
2885
2886 /**
2887  * ixgbevf_free_rx_resources - Free Rx Resources
2888  * @rx_ring: ring to clean the resources from
2889  *
2890  * Free all receive software resources
2891  **/
2892 void ixgbevf_free_rx_resources(struct ixgbevf_ring *rx_ring)
2893 {
2894         ixgbevf_clean_rx_ring(rx_ring);
2895
2896         vfree(rx_ring->rx_buffer_info);
2897         rx_ring->rx_buffer_info = NULL;
2898
2899         dma_free_coherent(rx_ring->dev, rx_ring->size, rx_ring->desc,
2900                           rx_ring->dma);
2901
2902         rx_ring->desc = NULL;
2903 }
2904
2905 /**
2906  * ixgbevf_free_all_rx_resources - Free Rx Resources for All Queues
2907  * @adapter: board private structure
2908  *
2909  * Free all receive software resources
2910  **/
2911 static void ixgbevf_free_all_rx_resources(struct ixgbevf_adapter *adapter)
2912 {
2913         int i;
2914
2915         for (i = 0; i < adapter->num_rx_queues; i++)
2916                 if (adapter->rx_ring[i]->desc)
2917                         ixgbevf_free_rx_resources(adapter->rx_ring[i]);
2918 }
2919
2920 /**
2921  * ixgbevf_open - Called when a network interface is made active
2922  * @netdev: network interface device structure
2923  *
2924  * Returns 0 on success, negative value on failure
2925  *
2926  * The open entry point is called when a network interface is made
2927  * active by the system (IFF_UP).  At this point all resources needed
2928  * for transmit and receive operations are allocated, the interrupt
2929  * handler is registered with the OS, the watchdog timer is started,
2930  * and the stack is notified that the interface is ready.
2931  **/
2932 static int ixgbevf_open(struct net_device *netdev)
2933 {
2934         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
2935         struct ixgbe_hw *hw = &adapter->hw;
2936         int err;
2937
2938         /* A previous failure to open the device because of a lack of
2939          * available MSIX vector resources may have reset the number
2940          * of msix vectors variable to zero.  The only way to recover
2941          * is to unload/reload the driver and hope that the system has
2942          * been able to recover some MSIX vector resources.
2943          */
2944         if (!adapter->num_msix_vectors)
2945                 return -ENOMEM;
2946
2947         /* disallow open during test */
2948         if (test_bit(__IXGBEVF_TESTING, &adapter->state))
2949                 return -EBUSY;
2950
2951         if (hw->adapter_stopped) {
2952                 ixgbevf_reset(adapter);
2953                 /* if adapter is still stopped then PF isn't up and
2954                  * the vf can't start. */
2955                 if (hw->adapter_stopped) {
2956                         err = IXGBE_ERR_MBX;
2957                         pr_err("Unable to start - perhaps the PF Driver isn't "
2958                                "up yet\n");
2959                         goto err_setup_reset;
2960                 }
2961         }
2962
2963         /* allocate transmit descriptors */
2964         err = ixgbevf_setup_all_tx_resources(adapter);
2965         if (err)
2966                 goto err_setup_tx;
2967
2968         /* allocate receive descriptors */
2969         err = ixgbevf_setup_all_rx_resources(adapter);
2970         if (err)
2971                 goto err_setup_rx;
2972
2973         ixgbevf_configure(adapter);
2974
2975         /*
2976          * Map the Tx/Rx rings to the vectors we were allotted.
2977          * if request_irq will be called in this function map_rings
2978          * must be called *before* up_complete
2979          */
2980         ixgbevf_map_rings_to_vectors(adapter);
2981
2982         ixgbevf_up_complete(adapter);
2983
2984         /* clear any pending interrupts, may auto mask */
2985         IXGBE_READ_REG(hw, IXGBE_VTEICR);
2986         err = ixgbevf_request_irq(adapter);
2987         if (err)
2988                 goto err_req_irq;
2989
2990         ixgbevf_irq_enable(adapter);
2991
2992         return 0;
2993
2994 err_req_irq:
2995         ixgbevf_down(adapter);
2996 err_setup_rx:
2997         ixgbevf_free_all_rx_resources(adapter);
2998 err_setup_tx:
2999         ixgbevf_free_all_tx_resources(adapter);
3000         ixgbevf_reset(adapter);
3001
3002 err_setup_reset:
3003
3004         return err;
3005 }
3006
3007 /**
3008  * ixgbevf_close - Disables a network interface
3009  * @netdev: network interface device structure
3010  *
3011  * Returns 0, this is not allowed to fail
3012  *
3013  * The close entry point is called when an interface is de-activated
3014  * by the OS.  The hardware is still under the drivers control, but
3015  * needs to be disabled.  A global MAC reset is issued to stop the
3016  * hardware, and all transmit and receive resources are freed.
3017  **/
3018 static int ixgbevf_close(struct net_device *netdev)
3019 {
3020         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3021
3022         ixgbevf_down(adapter);
3023         ixgbevf_free_irq(adapter);
3024
3025         ixgbevf_free_all_tx_resources(adapter);
3026         ixgbevf_free_all_rx_resources(adapter);
3027
3028         return 0;
3029 }
3030
3031 static void ixgbevf_queue_reset_subtask(struct ixgbevf_adapter *adapter)
3032 {
3033         struct net_device *dev = adapter->netdev;
3034
3035         if (!(adapter->flags & IXGBEVF_FLAG_QUEUE_RESET_REQUESTED))
3036                 return;
3037
3038         adapter->flags &= ~IXGBEVF_FLAG_QUEUE_RESET_REQUESTED;
3039
3040         /* if interface is down do nothing */
3041         if (test_bit(__IXGBEVF_DOWN, &adapter->state) ||
3042             test_bit(__IXGBEVF_RESETTING, &adapter->state))
3043                 return;
3044
3045         /* Hardware has to reinitialize queues and interrupts to
3046          * match packet buffer alignment. Unfortunately, the
3047          * hardware is not flexible enough to do this dynamically.
3048          */
3049         if (netif_running(dev))
3050                 ixgbevf_close(dev);
3051
3052         ixgbevf_clear_interrupt_scheme(adapter);
3053         ixgbevf_init_interrupt_scheme(adapter);
3054
3055         if (netif_running(dev))
3056                 ixgbevf_open(dev);
3057 }
3058
3059 static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring,
3060                                 u32 vlan_macip_lens, u32 type_tucmd,
3061                                 u32 mss_l4len_idx)
3062 {
3063         struct ixgbe_adv_tx_context_desc *context_desc;
3064         u16 i = tx_ring->next_to_use;
3065
3066         context_desc = IXGBEVF_TX_CTXTDESC(tx_ring, i);
3067
3068         i++;
3069         tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
3070
3071         /* set bits to identify this as an advanced context descriptor */
3072         type_tucmd |= IXGBE_TXD_CMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
3073
3074         context_desc->vlan_macip_lens   = cpu_to_le32(vlan_macip_lens);
3075         context_desc->seqnum_seed       = 0;
3076         context_desc->type_tucmd_mlhl   = cpu_to_le32(type_tucmd);
3077         context_desc->mss_l4len_idx     = cpu_to_le32(mss_l4len_idx);
3078 }
3079
3080 static int ixgbevf_tso(struct ixgbevf_ring *tx_ring,
3081                        struct ixgbevf_tx_buffer *first,
3082                        u8 *hdr_len)
3083 {
3084         struct sk_buff *skb = first->skb;
3085         u32 vlan_macip_lens, type_tucmd;
3086         u32 mss_l4len_idx, l4len;
3087         int err;
3088
3089         if (skb->ip_summed != CHECKSUM_PARTIAL)
3090                 return 0;
3091
3092         if (!skb_is_gso(skb))
3093                 return 0;
3094
3095         err = skb_cow_head(skb, 0);
3096         if (err < 0)
3097                 return err;
3098
3099         /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
3100         type_tucmd = IXGBE_ADVTXD_TUCMD_L4T_TCP;
3101
3102         if (first->protocol == htons(ETH_P_IP)) {
3103                 struct iphdr *iph = ip_hdr(skb);
3104                 iph->tot_len = 0;
3105                 iph->check = 0;
3106                 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3107                                                          iph->daddr, 0,
3108                                                          IPPROTO_TCP,
3109                                                          0);
3110                 type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
3111                 first->tx_flags |= IXGBE_TX_FLAGS_TSO |
3112                                    IXGBE_TX_FLAGS_CSUM |
3113                                    IXGBE_TX_FLAGS_IPV4;
3114         } else if (skb_is_gso_v6(skb)) {
3115                 ipv6_hdr(skb)->payload_len = 0;
3116                 tcp_hdr(skb)->check =
3117                     ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
3118                                      &ipv6_hdr(skb)->daddr,
3119                                      0, IPPROTO_TCP, 0);
3120                 first->tx_flags |= IXGBE_TX_FLAGS_TSO |
3121                                    IXGBE_TX_FLAGS_CSUM;
3122         }
3123
3124         /* compute header lengths */
3125         l4len = tcp_hdrlen(skb);
3126         *hdr_len += l4len;
3127         *hdr_len = skb_transport_offset(skb) + l4len;
3128
3129         /* update gso size and bytecount with header size */
3130         first->gso_segs = skb_shinfo(skb)->gso_segs;
3131         first->bytecount += (first->gso_segs - 1) * *hdr_len;
3132
3133         /* mss_l4len_id: use 1 as index for TSO */
3134         mss_l4len_idx = l4len << IXGBE_ADVTXD_L4LEN_SHIFT;
3135         mss_l4len_idx |= skb_shinfo(skb)->gso_size << IXGBE_ADVTXD_MSS_SHIFT;
3136         mss_l4len_idx |= 1 << IXGBE_ADVTXD_IDX_SHIFT;
3137
3138         /* vlan_macip_lens: HEADLEN, MACLEN, VLAN tag */
3139         vlan_macip_lens = skb_network_header_len(skb);
3140         vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
3141         vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
3142
3143         ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
3144                             type_tucmd, mss_l4len_idx);
3145
3146         return 1;
3147 }
3148
3149 static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring,
3150                             struct ixgbevf_tx_buffer *first)
3151 {
3152         struct sk_buff *skb = first->skb;
3153         u32 vlan_macip_lens = 0;
3154         u32 mss_l4len_idx = 0;
3155         u32 type_tucmd = 0;
3156
3157         if (skb->ip_summed == CHECKSUM_PARTIAL) {
3158                 u8 l4_hdr = 0;
3159                 switch (first->protocol) {
3160                 case htons(ETH_P_IP):
3161                         vlan_macip_lens |= skb_network_header_len(skb);
3162                         type_tucmd |= IXGBE_ADVTXD_TUCMD_IPV4;
3163                         l4_hdr = ip_hdr(skb)->protocol;
3164                         break;
3165                 case htons(ETH_P_IPV6):
3166                         vlan_macip_lens |= skb_network_header_len(skb);
3167                         l4_hdr = ipv6_hdr(skb)->nexthdr;
3168                         break;
3169                 default:
3170                         if (unlikely(net_ratelimit())) {
3171                                 dev_warn(tx_ring->dev,
3172                                  "partial checksum but proto=%x!\n",
3173                                  first->protocol);
3174                         }
3175                         break;
3176                 }
3177
3178                 switch (l4_hdr) {
3179                 case IPPROTO_TCP:
3180                         type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
3181                         mss_l4len_idx = tcp_hdrlen(skb) <<
3182                                         IXGBE_ADVTXD_L4LEN_SHIFT;
3183                         break;
3184                 case IPPROTO_SCTP:
3185                         type_tucmd |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
3186                         mss_l4len_idx = sizeof(struct sctphdr) <<
3187                                         IXGBE_ADVTXD_L4LEN_SHIFT;
3188                         break;
3189                 case IPPROTO_UDP:
3190                         mss_l4len_idx = sizeof(struct udphdr) <<
3191                                         IXGBE_ADVTXD_L4LEN_SHIFT;
3192                         break;
3193                 default:
3194                         if (unlikely(net_ratelimit())) {
3195                                 dev_warn(tx_ring->dev,
3196                                  "partial checksum but l4 proto=%x!\n",
3197                                  l4_hdr);
3198                         }
3199                         break;
3200                 }
3201
3202                 /* update TX checksum flag */
3203                 first->tx_flags |= IXGBE_TX_FLAGS_CSUM;
3204         }
3205
3206         /* vlan_macip_lens: MACLEN, VLAN tag */
3207         vlan_macip_lens |= skb_network_offset(skb) << IXGBE_ADVTXD_MACLEN_SHIFT;
3208         vlan_macip_lens |= first->tx_flags & IXGBE_TX_FLAGS_VLAN_MASK;
3209
3210         ixgbevf_tx_ctxtdesc(tx_ring, vlan_macip_lens,
3211                             type_tucmd, mss_l4len_idx);
3212 }
3213
3214 static __le32 ixgbevf_tx_cmd_type(u32 tx_flags)
3215 {
3216         /* set type for advanced descriptor with frame checksum insertion */
3217         __le32 cmd_type = cpu_to_le32(IXGBE_ADVTXD_DTYP_DATA |
3218                                       IXGBE_ADVTXD_DCMD_IFCS |
3219                                       IXGBE_ADVTXD_DCMD_DEXT);
3220
3221         /* set HW vlan bit if vlan is present */
3222         if (tx_flags & IXGBE_TX_FLAGS_VLAN)
3223                 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_VLE);
3224
3225         /* set segmentation enable bits for TSO/FSO */
3226         if (tx_flags & IXGBE_TX_FLAGS_TSO)
3227                 cmd_type |= cpu_to_le32(IXGBE_ADVTXD_DCMD_TSE);
3228
3229         return cmd_type;
3230 }
3231
3232 static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc,
3233                                      u32 tx_flags, unsigned int paylen)
3234 {
3235         __le32 olinfo_status = cpu_to_le32(paylen << IXGBE_ADVTXD_PAYLEN_SHIFT);
3236
3237         /* enable L4 checksum for TSO and TX checksum offload */
3238         if (tx_flags & IXGBE_TX_FLAGS_CSUM)
3239                 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_TXSM);
3240
3241         /* enble IPv4 checksum for TSO */
3242         if (tx_flags & IXGBE_TX_FLAGS_IPV4)
3243                 olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_POPTS_IXSM);
3244
3245         /* use index 1 context for TSO/FSO/FCOE */
3246         if (tx_flags & IXGBE_TX_FLAGS_TSO)
3247                 olinfo_status |= cpu_to_le32(1 << IXGBE_ADVTXD_IDX_SHIFT);
3248
3249         /* Check Context must be set if Tx switch is enabled, which it
3250          * always is for case where virtual functions are running
3251          */
3252         olinfo_status |= cpu_to_le32(IXGBE_ADVTXD_CC);
3253
3254         tx_desc->read.olinfo_status = olinfo_status;
3255 }
3256
3257 static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring,
3258                            struct ixgbevf_tx_buffer *first,
3259                            const u8 hdr_len)
3260 {
3261         dma_addr_t dma;
3262         struct sk_buff *skb = first->skb;
3263         struct ixgbevf_tx_buffer *tx_buffer;
3264         union ixgbe_adv_tx_desc *tx_desc;
3265         struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
3266         unsigned int data_len = skb->data_len;
3267         unsigned int size = skb_headlen(skb);
3268         unsigned int paylen = skb->len - hdr_len;
3269         u32 tx_flags = first->tx_flags;
3270         __le32 cmd_type;
3271         u16 i = tx_ring->next_to_use;
3272
3273         tx_desc = IXGBEVF_TX_DESC(tx_ring, i);
3274
3275         ixgbevf_tx_olinfo_status(tx_desc, tx_flags, paylen);
3276         cmd_type = ixgbevf_tx_cmd_type(tx_flags);
3277
3278         dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
3279         if (dma_mapping_error(tx_ring->dev, dma))
3280                 goto dma_error;
3281
3282         /* record length, and DMA address */
3283         dma_unmap_len_set(first, len, size);
3284         dma_unmap_addr_set(first, dma, dma);
3285
3286         tx_desc->read.buffer_addr = cpu_to_le64(dma);
3287
3288         for (;;) {
3289                 while (unlikely(size > IXGBE_MAX_DATA_PER_TXD)) {
3290                         tx_desc->read.cmd_type_len =
3291                                 cmd_type | cpu_to_le32(IXGBE_MAX_DATA_PER_TXD);
3292
3293                         i++;
3294                         tx_desc++;
3295                         if (i == tx_ring->count) {
3296                                 tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
3297                                 i = 0;
3298                         }
3299
3300                         dma += IXGBE_MAX_DATA_PER_TXD;
3301                         size -= IXGBE_MAX_DATA_PER_TXD;
3302
3303                         tx_desc->read.buffer_addr = cpu_to_le64(dma);
3304                         tx_desc->read.olinfo_status = 0;
3305                 }
3306
3307                 if (likely(!data_len))
3308                         break;
3309
3310                 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
3311
3312                 i++;
3313                 tx_desc++;
3314                 if (i == tx_ring->count) {
3315                         tx_desc = IXGBEVF_TX_DESC(tx_ring, 0);
3316                         i = 0;
3317                 }
3318
3319                 size = skb_frag_size(frag);
3320                 data_len -= size;
3321
3322                 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
3323                                        DMA_TO_DEVICE);
3324                 if (dma_mapping_error(tx_ring->dev, dma))
3325                         goto dma_error;
3326
3327                 tx_buffer = &tx_ring->tx_buffer_info[i];
3328                 dma_unmap_len_set(tx_buffer, len, size);
3329                 dma_unmap_addr_set(tx_buffer, dma, dma);
3330
3331                 tx_desc->read.buffer_addr = cpu_to_le64(dma);
3332                 tx_desc->read.olinfo_status = 0;
3333
3334                 frag++;
3335         }
3336
3337         /* write last descriptor with RS and EOP bits */
3338         cmd_type |= cpu_to_le32(size) | cpu_to_le32(IXGBE_TXD_CMD);
3339         tx_desc->read.cmd_type_len = cmd_type;
3340
3341         /* set the timestamp */
3342         first->time_stamp = jiffies;
3343
3344         /* Force memory writes to complete before letting h/w know there
3345          * are new descriptors to fetch.  (Only applicable for weak-ordered
3346          * memory model archs, such as IA-64).
3347          *
3348          * We also need this memory barrier (wmb) to make certain all of the
3349          * status bits have been updated before next_to_watch is written.
3350          */
3351         wmb();
3352
3353         /* set next_to_watch value indicating a packet is present */
3354         first->next_to_watch = tx_desc;
3355
3356         i++;
3357         if (i == tx_ring->count)
3358                 i = 0;
3359
3360         tx_ring->next_to_use = i;
3361
3362         /* notify HW of packet */
3363         ixgbevf_write_tail(tx_ring, i);
3364
3365         return;
3366 dma_error:
3367         dev_err(tx_ring->dev, "TX DMA map failed\n");
3368
3369         /* clear dma mappings for failed tx_buffer_info map */
3370         for (;;) {
3371                 tx_buffer = &tx_ring->tx_buffer_info[i];
3372                 ixgbevf_unmap_and_free_tx_resource(tx_ring, tx_buffer);
3373                 if (tx_buffer == first)
3374                         break;
3375                 if (i == 0)
3376                         i = tx_ring->count;
3377                 i--;
3378         }
3379
3380         tx_ring->next_to_use = i;
3381 }
3382
3383 static int __ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
3384 {
3385         netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
3386         /* Herbert's original patch had:
3387          *  smp_mb__after_netif_stop_queue();
3388          * but since that doesn't exist yet, just open code it. */
3389         smp_mb();
3390
3391         /* We need to check again in a case another CPU has just
3392          * made room available. */
3393         if (likely(ixgbevf_desc_unused(tx_ring) < size))
3394                 return -EBUSY;
3395
3396         /* A reprieve! - use start_queue because it doesn't call schedule */
3397         netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
3398         ++tx_ring->tx_stats.restart_queue;
3399
3400         return 0;
3401 }
3402
3403 static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size)
3404 {
3405         if (likely(ixgbevf_desc_unused(tx_ring) >= size))
3406                 return 0;
3407         return __ixgbevf_maybe_stop_tx(tx_ring, size);
3408 }
3409
3410 static int ixgbevf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
3411 {
3412         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3413         struct ixgbevf_tx_buffer *first;
3414         struct ixgbevf_ring *tx_ring;
3415         int tso;
3416         u32 tx_flags = 0;
3417         u16 count = TXD_USE_COUNT(skb_headlen(skb));
3418 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3419         unsigned short f;
3420 #endif
3421         u8 hdr_len = 0;
3422         u8 *dst_mac = skb_header_pointer(skb, 0, 0, NULL);
3423
3424         if (!dst_mac || is_link_local_ether_addr(dst_mac)) {
3425                 dev_kfree_skb(skb);
3426                 return NETDEV_TX_OK;
3427         }
3428
3429         tx_ring = adapter->tx_ring[skb->queue_mapping];
3430
3431         /*
3432          * need: 1 descriptor per page * PAGE_SIZE/IXGBE_MAX_DATA_PER_TXD,
3433          *       + 1 desc for skb_headlen/IXGBE_MAX_DATA_PER_TXD,
3434          *       + 2 desc gap to keep tail from touching head,
3435          *       + 1 desc for context descriptor,
3436          * otherwise try next time
3437          */
3438 #if PAGE_SIZE > IXGBE_MAX_DATA_PER_TXD
3439         for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
3440                 count += TXD_USE_COUNT(skb_shinfo(skb)->frags[f].size);
3441 #else
3442         count += skb_shinfo(skb)->nr_frags;
3443 #endif
3444         if (ixgbevf_maybe_stop_tx(tx_ring, count + 3)) {
3445                 tx_ring->tx_stats.tx_busy++;
3446                 return NETDEV_TX_BUSY;
3447         }
3448
3449         /* record the location of the first descriptor for this packet */
3450         first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
3451         first->skb = skb;
3452         first->bytecount = skb->len;
3453         first->gso_segs = 1;
3454
3455         if (vlan_tx_tag_present(skb)) {
3456                 tx_flags |= vlan_tx_tag_get(skb);
3457                 tx_flags <<= IXGBE_TX_FLAGS_VLAN_SHIFT;
3458                 tx_flags |= IXGBE_TX_FLAGS_VLAN;
3459         }
3460
3461         /* record initial flags and protocol */
3462         first->tx_flags = tx_flags;
3463         first->protocol = vlan_get_protocol(skb);
3464
3465         tso = ixgbevf_tso(tx_ring, first, &hdr_len);
3466         if (tso < 0)
3467                 goto out_drop;
3468         else if (!tso)
3469                 ixgbevf_tx_csum(tx_ring, first);
3470
3471         ixgbevf_tx_map(tx_ring, first, hdr_len);
3472
3473         ixgbevf_maybe_stop_tx(tx_ring, DESC_NEEDED);
3474
3475         return NETDEV_TX_OK;
3476
3477 out_drop:
3478         dev_kfree_skb_any(first->skb);
3479         first->skb = NULL;
3480
3481         return NETDEV_TX_OK;
3482 }
3483
3484 /**
3485  * ixgbevf_set_mac - Change the Ethernet Address of the NIC
3486  * @netdev: network interface device structure
3487  * @p: pointer to an address structure
3488  *
3489  * Returns 0 on success, negative on failure
3490  **/
3491 static int ixgbevf_set_mac(struct net_device *netdev, void *p)
3492 {
3493         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3494         struct ixgbe_hw *hw = &adapter->hw;
3495         struct sockaddr *addr = p;
3496
3497         if (!is_valid_ether_addr(addr->sa_data))
3498                 return -EADDRNOTAVAIL;
3499
3500         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3501         memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3502
3503         spin_lock_bh(&adapter->mbx_lock);
3504
3505         hw->mac.ops.set_rar(hw, 0, hw->mac.addr, 0);
3506
3507         spin_unlock_bh(&adapter->mbx_lock);
3508
3509         return 0;
3510 }
3511
3512 /**
3513  * ixgbevf_change_mtu - Change the Maximum Transfer Unit
3514  * @netdev: network interface device structure
3515  * @new_mtu: new value for maximum frame size
3516  *
3517  * Returns 0 on success, negative on failure
3518  **/
3519 static int ixgbevf_change_mtu(struct net_device *netdev, int new_mtu)
3520 {
3521         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3522         struct ixgbe_hw *hw = &adapter->hw;
3523         int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
3524         int max_possible_frame = MAXIMUM_ETHERNET_VLAN_SIZE;
3525
3526         switch (adapter->hw.api_version) {
3527         case ixgbe_mbox_api_11:
3528                 max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
3529                 break;
3530         default:
3531                 if (adapter->hw.mac.type != ixgbe_mac_82599_vf)
3532                         max_possible_frame = IXGBE_MAX_JUMBO_FRAME_SIZE;
3533                 break;
3534         }
3535
3536         /* MTU < 68 is an error and causes problems on some kernels */
3537         if ((new_mtu < 68) || (max_frame > max_possible_frame))
3538                 return -EINVAL;
3539
3540         hw_dbg(hw, "changing MTU from %d to %d\n",
3541                netdev->mtu, new_mtu);
3542         /* must set new MTU before calling down or up */
3543         netdev->mtu = new_mtu;
3544
3545         /* notify the PF of our intent to use this size of frame */
3546         ixgbevf_rlpml_set_vf(hw, max_frame);
3547
3548         return 0;
3549 }
3550
3551 #ifdef CONFIG_NET_POLL_CONTROLLER
3552 /* Polling 'interrupt' - used by things like netconsole to send skbs
3553  * without having to re-enable interrupts. It's not called while
3554  * the interrupt routine is executing.
3555  */
3556 static void ixgbevf_netpoll(struct net_device *netdev)
3557 {
3558         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3559         int i;
3560
3561         /* if interface is down do nothing */
3562         if (test_bit(__IXGBEVF_DOWN, &adapter->state))
3563                 return;
3564         for (i = 0; i < adapter->num_rx_queues; i++)
3565                 ixgbevf_msix_clean_rings(0, adapter->q_vector[i]);
3566 }
3567 #endif /* CONFIG_NET_POLL_CONTROLLER */
3568
3569 static int ixgbevf_suspend(struct pci_dev *pdev, pm_message_t state)
3570 {
3571         struct net_device *netdev = pci_get_drvdata(pdev);
3572         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3573 #ifdef CONFIG_PM
3574         int retval = 0;
3575 #endif
3576
3577         netif_device_detach(netdev);
3578
3579         if (netif_running(netdev)) {
3580                 rtnl_lock();
3581                 ixgbevf_down(adapter);
3582                 ixgbevf_free_irq(adapter);
3583                 ixgbevf_free_all_tx_resources(adapter);
3584                 ixgbevf_free_all_rx_resources(adapter);
3585                 rtnl_unlock();
3586         }
3587
3588         ixgbevf_clear_interrupt_scheme(adapter);
3589
3590 #ifdef CONFIG_PM
3591         retval = pci_save_state(pdev);
3592         if (retval)
3593                 return retval;
3594
3595 #endif
3596         if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state))
3597                 pci_disable_device(pdev);
3598
3599         return 0;
3600 }
3601
3602 #ifdef CONFIG_PM
3603 static int ixgbevf_resume(struct pci_dev *pdev)
3604 {
3605         struct net_device *netdev = pci_get_drvdata(pdev);
3606         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3607         u32 err;
3608
3609         pci_restore_state(pdev);
3610         /*
3611          * pci_restore_state clears dev->state_saved so call
3612          * pci_save_state to restore it.
3613          */
3614         pci_save_state(pdev);
3615
3616         err = pci_enable_device_mem(pdev);
3617         if (err) {
3618                 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n");
3619                 return err;
3620         }
3621         smp_mb__before_atomic();
3622         clear_bit(__IXGBEVF_DISABLED, &adapter->state);
3623         pci_set_master(pdev);
3624
3625         ixgbevf_reset(adapter);
3626
3627         rtnl_lock();
3628         err = ixgbevf_init_interrupt_scheme(adapter);
3629         rtnl_unlock();
3630         if (err) {
3631                 dev_err(&pdev->dev, "Cannot initialize interrupts\n");
3632                 return err;
3633         }
3634
3635         if (netif_running(netdev)) {
3636                 err = ixgbevf_open(netdev);
3637                 if (err)
3638                         return err;
3639         }
3640
3641         netif_device_attach(netdev);
3642
3643         return err;
3644 }
3645
3646 #endif /* CONFIG_PM */
3647 static void ixgbevf_shutdown(struct pci_dev *pdev)
3648 {
3649         ixgbevf_suspend(pdev, PMSG_SUSPEND);
3650 }
3651
3652 static struct rtnl_link_stats64 *ixgbevf_get_stats(struct net_device *netdev,
3653                                                 struct rtnl_link_stats64 *stats)
3654 {
3655         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3656         unsigned int start;
3657         u64 bytes, packets;
3658         const struct ixgbevf_ring *ring;
3659         int i;
3660
3661         ixgbevf_update_stats(adapter);
3662
3663         stats->multicast = adapter->stats.vfmprc - adapter->stats.base_vfmprc;
3664
3665         for (i = 0; i < adapter->num_rx_queues; i++) {
3666                 ring = adapter->rx_ring[i];
3667                 do {
3668                         start = u64_stats_fetch_begin_irq(&ring->syncp);
3669                         bytes = ring->stats.bytes;
3670                         packets = ring->stats.packets;
3671                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
3672                 stats->rx_bytes += bytes;
3673                 stats->rx_packets += packets;
3674         }
3675
3676         for (i = 0; i < adapter->num_tx_queues; i++) {
3677                 ring = adapter->tx_ring[i];
3678                 do {
3679                         start = u64_stats_fetch_begin_irq(&ring->syncp);
3680                         bytes = ring->stats.bytes;
3681                         packets = ring->stats.packets;
3682                 } while (u64_stats_fetch_retry_irq(&ring->syncp, start));
3683                 stats->tx_bytes += bytes;
3684                 stats->tx_packets += packets;
3685         }
3686
3687         return stats;
3688 }
3689
3690 static const struct net_device_ops ixgbevf_netdev_ops = {
3691         .ndo_open               = ixgbevf_open,
3692         .ndo_stop               = ixgbevf_close,
3693         .ndo_start_xmit         = ixgbevf_xmit_frame,
3694         .ndo_set_rx_mode        = ixgbevf_set_rx_mode,
3695         .ndo_get_stats64        = ixgbevf_get_stats,
3696         .ndo_validate_addr      = eth_validate_addr,
3697         .ndo_set_mac_address    = ixgbevf_set_mac,
3698         .ndo_change_mtu         = ixgbevf_change_mtu,
3699         .ndo_tx_timeout         = ixgbevf_tx_timeout,
3700         .ndo_vlan_rx_add_vid    = ixgbevf_vlan_rx_add_vid,
3701         .ndo_vlan_rx_kill_vid   = ixgbevf_vlan_rx_kill_vid,
3702 #ifdef CONFIG_NET_RX_BUSY_POLL
3703         .ndo_busy_poll          = ixgbevf_busy_poll_recv,
3704 #endif
3705 #ifdef CONFIG_NET_POLL_CONTROLLER
3706         .ndo_poll_controller    = ixgbevf_netpoll,
3707 #endif
3708 };
3709
3710 static void ixgbevf_assign_netdev_ops(struct net_device *dev)
3711 {
3712         dev->netdev_ops = &ixgbevf_netdev_ops;
3713         ixgbevf_set_ethtool_ops(dev);
3714         dev->watchdog_timeo = 5 * HZ;
3715 }
3716
3717 /**
3718  * ixgbevf_probe - Device Initialization Routine
3719  * @pdev: PCI device information struct
3720  * @ent: entry in ixgbevf_pci_tbl
3721  *
3722  * Returns 0 on success, negative on failure
3723  *
3724  * ixgbevf_probe initializes an adapter identified by a pci_dev structure.
3725  * The OS initialization, configuring of the adapter private structure,
3726  * and a hardware reset occur.
3727  **/
3728 static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
3729 {
3730         struct net_device *netdev;
3731         struct ixgbevf_adapter *adapter = NULL;
3732         struct ixgbe_hw *hw = NULL;
3733         const struct ixgbevf_info *ii = ixgbevf_info_tbl[ent->driver_data];
3734         int err, pci_using_dac;
3735         bool disable_dev = false;
3736
3737         err = pci_enable_device(pdev);
3738         if (err)
3739                 return err;
3740
3741         if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
3742                 pci_using_dac = 1;
3743         } else {
3744                 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
3745                 if (err) {
3746                         dev_err(&pdev->dev, "No usable DMA "
3747                                 "configuration, aborting\n");
3748                         goto err_dma;
3749                 }
3750                 pci_using_dac = 0;
3751         }
3752
3753         err = pci_request_regions(pdev, ixgbevf_driver_name);
3754         if (err) {
3755                 dev_err(&pdev->dev, "pci_request_regions failed 0x%x\n", err);
3756                 goto err_pci_reg;
3757         }
3758
3759         pci_set_master(pdev);
3760
3761         netdev = alloc_etherdev_mq(sizeof(struct ixgbevf_adapter),
3762                                    MAX_TX_QUEUES);
3763         if (!netdev) {
3764                 err = -ENOMEM;
3765                 goto err_alloc_etherdev;
3766         }
3767
3768         SET_NETDEV_DEV(netdev, &pdev->dev);
3769
3770         adapter = netdev_priv(netdev);
3771
3772         adapter->netdev = netdev;
3773         adapter->pdev = pdev;
3774         hw = &adapter->hw;
3775         hw->back = adapter;
3776         adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3777
3778         /*
3779          * call save state here in standalone driver because it relies on
3780          * adapter struct to exist, and needs to call netdev_priv
3781          */
3782         pci_save_state(pdev);
3783
3784         hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
3785                               pci_resource_len(pdev, 0));
3786         adapter->io_addr = hw->hw_addr;
3787         if (!hw->hw_addr) {
3788                 err = -EIO;
3789                 goto err_ioremap;
3790         }
3791
3792         ixgbevf_assign_netdev_ops(netdev);
3793
3794         /* Setup hw api */
3795         memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops));
3796         hw->mac.type  = ii->mac;
3797
3798         memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops,
3799                sizeof(struct ixgbe_mbx_operations));
3800
3801         /* setup the private structure */
3802         err = ixgbevf_sw_init(adapter);
3803         if (err)
3804                 goto err_sw_init;
3805
3806         /* The HW MAC address was set and/or determined in sw_init */
3807         if (!is_valid_ether_addr(netdev->dev_addr)) {
3808                 pr_err("invalid MAC address\n");
3809                 err = -EIO;
3810                 goto err_sw_init;
3811         }
3812
3813         netdev->hw_features = NETIF_F_SG |
3814                            NETIF_F_IP_CSUM |
3815                            NETIF_F_IPV6_CSUM |
3816                            NETIF_F_TSO |
3817                            NETIF_F_TSO6 |
3818                            NETIF_F_RXCSUM;
3819
3820         netdev->features = netdev->hw_features |
3821                            NETIF_F_HW_VLAN_CTAG_TX |
3822                            NETIF_F_HW_VLAN_CTAG_RX |
3823                            NETIF_F_HW_VLAN_CTAG_FILTER;
3824
3825         netdev->vlan_features |= NETIF_F_TSO;
3826         netdev->vlan_features |= NETIF_F_TSO6;
3827         netdev->vlan_features |= NETIF_F_IP_CSUM;
3828         netdev->vlan_features |= NETIF_F_IPV6_CSUM;
3829         netdev->vlan_features |= NETIF_F_SG;
3830
3831         if (pci_using_dac)
3832                 netdev->features |= NETIF_F_HIGHDMA;
3833
3834         netdev->priv_flags |= IFF_UNICAST_FLT;
3835
3836         init_timer(&adapter->watchdog_timer);
3837         adapter->watchdog_timer.function = ixgbevf_watchdog;
3838         adapter->watchdog_timer.data = (unsigned long)adapter;
3839
3840         if (IXGBE_REMOVED(hw->hw_addr)) {
3841                 err = -EIO;
3842                 goto err_sw_init;
3843         }
3844         INIT_WORK(&adapter->reset_task, ixgbevf_reset_task);
3845         INIT_WORK(&adapter->watchdog_task, ixgbevf_watchdog_task);
3846         set_bit(__IXGBEVF_WORK_INIT, &adapter->state);
3847
3848         err = ixgbevf_init_interrupt_scheme(adapter);
3849         if (err)
3850                 goto err_sw_init;
3851
3852         strcpy(netdev->name, "eth%d");
3853
3854         err = register_netdev(netdev);
3855         if (err)
3856                 goto err_register;
3857
3858         pci_set_drvdata(pdev, netdev);
3859         netif_carrier_off(netdev);
3860
3861         ixgbevf_init_last_counter_stats(adapter);
3862
3863         /* print the VF info */
3864         dev_info(&pdev->dev, "%pM\n", netdev->dev_addr);
3865         dev_info(&pdev->dev, "MAC: %d\n", hw->mac.type);
3866
3867         switch (hw->mac.type) {
3868         case ixgbe_mac_X550_vf:
3869                 dev_info(&pdev->dev, "Intel(R) X550 Virtual Function\n");
3870                 break;
3871         case ixgbe_mac_X540_vf:
3872                 dev_info(&pdev->dev, "Intel(R) X540 Virtual Function\n");
3873                 break;
3874         case ixgbe_mac_82599_vf:
3875         default:
3876                 dev_info(&pdev->dev, "Intel(R) 82599 Virtual Function\n");
3877                 break;
3878         }
3879
3880         return 0;
3881
3882 err_register:
3883         ixgbevf_clear_interrupt_scheme(adapter);
3884 err_sw_init:
3885         ixgbevf_reset_interrupt_capability(adapter);
3886         iounmap(adapter->io_addr);
3887 err_ioremap:
3888         disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state);
3889         free_netdev(netdev);
3890 err_alloc_etherdev:
3891         pci_release_regions(pdev);
3892 err_pci_reg:
3893 err_dma:
3894         if (!adapter || disable_dev)
3895                 pci_disable_device(pdev);
3896         return err;
3897 }
3898
3899 /**
3900  * ixgbevf_remove - Device Removal Routine
3901  * @pdev: PCI device information struct
3902  *
3903  * ixgbevf_remove is called by the PCI subsystem to alert the driver
3904  * that it should release a PCI device.  The could be caused by a
3905  * Hot-Plug event, or because the driver is going to be removed from
3906  * memory.
3907  **/
3908 static void ixgbevf_remove(struct pci_dev *pdev)
3909 {
3910         struct net_device *netdev = pci_get_drvdata(pdev);
3911         struct ixgbevf_adapter *adapter;
3912         bool disable_dev;
3913
3914         if (!netdev)
3915                 return;
3916
3917         adapter = netdev_priv(netdev);
3918
3919         set_bit(__IXGBEVF_REMOVING, &adapter->state);
3920
3921         del_timer_sync(&adapter->watchdog_timer);
3922
3923         cancel_work_sync(&adapter->reset_task);
3924         cancel_work_sync(&adapter->watchdog_task);
3925
3926         if (netdev->reg_state == NETREG_REGISTERED)
3927                 unregister_netdev(netdev);
3928
3929         ixgbevf_clear_interrupt_scheme(adapter);
3930         ixgbevf_reset_interrupt_capability(adapter);
3931
3932         iounmap(adapter->io_addr);
3933         pci_release_regions(pdev);
3934
3935         hw_dbg(&adapter->hw, "Remove complete\n");
3936
3937         disable_dev = !test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state);
3938         free_netdev(netdev);
3939
3940         if (disable_dev)
3941                 pci_disable_device(pdev);
3942 }
3943
3944 /**
3945  * ixgbevf_io_error_detected - called when PCI error is detected
3946  * @pdev: Pointer to PCI device
3947  * @state: The current pci connection state
3948  *
3949  * This function is called after a PCI bus error affecting
3950  * this device has been detected.
3951  */
3952 static pci_ers_result_t ixgbevf_io_error_detected(struct pci_dev *pdev,
3953                                                   pci_channel_state_t state)
3954 {
3955         struct net_device *netdev = pci_get_drvdata(pdev);
3956         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3957
3958         if (!test_bit(__IXGBEVF_WORK_INIT, &adapter->state))
3959                 return PCI_ERS_RESULT_DISCONNECT;
3960
3961         rtnl_lock();
3962         netif_device_detach(netdev);
3963
3964         if (state == pci_channel_io_perm_failure) {
3965                 rtnl_unlock();
3966                 return PCI_ERS_RESULT_DISCONNECT;
3967         }
3968
3969         if (netif_running(netdev))
3970                 ixgbevf_down(adapter);
3971
3972         if (!test_and_set_bit(__IXGBEVF_DISABLED, &adapter->state))
3973                 pci_disable_device(pdev);
3974         rtnl_unlock();
3975
3976         /* Request a slot slot reset. */
3977         return PCI_ERS_RESULT_NEED_RESET;
3978 }
3979
3980 /**
3981  * ixgbevf_io_slot_reset - called after the pci bus has been reset.
3982  * @pdev: Pointer to PCI device
3983  *
3984  * Restart the card from scratch, as if from a cold-boot. Implementation
3985  * resembles the first-half of the ixgbevf_resume routine.
3986  */
3987 static pci_ers_result_t ixgbevf_io_slot_reset(struct pci_dev *pdev)
3988 {
3989         struct net_device *netdev = pci_get_drvdata(pdev);
3990         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
3991
3992         if (pci_enable_device_mem(pdev)) {
3993                 dev_err(&pdev->dev,
3994                         "Cannot re-enable PCI device after reset.\n");
3995                 return PCI_ERS_RESULT_DISCONNECT;
3996         }
3997
3998         smp_mb__before_atomic();
3999         clear_bit(__IXGBEVF_DISABLED, &adapter->state);
4000         pci_set_master(pdev);
4001
4002         ixgbevf_reset(adapter);
4003
4004         return PCI_ERS_RESULT_RECOVERED;
4005 }
4006
4007 /**
4008  * ixgbevf_io_resume - called when traffic can start flowing again.
4009  * @pdev: Pointer to PCI device
4010  *
4011  * This callback is called when the error recovery driver tells us that
4012  * its OK to resume normal operation. Implementation resembles the
4013  * second-half of the ixgbevf_resume routine.
4014  */
4015 static void ixgbevf_io_resume(struct pci_dev *pdev)
4016 {
4017         struct net_device *netdev = pci_get_drvdata(pdev);
4018         struct ixgbevf_adapter *adapter = netdev_priv(netdev);
4019
4020         if (netif_running(netdev))
4021                 ixgbevf_up(adapter);
4022
4023         netif_device_attach(netdev);
4024 }
4025
4026 /* PCI Error Recovery (ERS) */
4027 static const struct pci_error_handlers ixgbevf_err_handler = {
4028         .error_detected = ixgbevf_io_error_detected,
4029         .slot_reset = ixgbevf_io_slot_reset,
4030         .resume = ixgbevf_io_resume,
4031 };
4032
4033 static struct pci_driver ixgbevf_driver = {
4034         .name     = ixgbevf_driver_name,
4035         .id_table = ixgbevf_pci_tbl,
4036         .probe    = ixgbevf_probe,
4037         .remove   = ixgbevf_remove,
4038 #ifdef CONFIG_PM
4039         /* Power Management Hooks */
4040         .suspend  = ixgbevf_suspend,
4041         .resume   = ixgbevf_resume,
4042 #endif
4043         .shutdown = ixgbevf_shutdown,
4044         .err_handler = &ixgbevf_err_handler
4045 };
4046
4047 /**
4048  * ixgbevf_init_module - Driver Registration Routine
4049  *
4050  * ixgbevf_init_module is the first routine called when the driver is
4051  * loaded. All it does is register with the PCI subsystem.
4052  **/
4053 static int __init ixgbevf_init_module(void)
4054 {
4055         int ret;
4056         pr_info("%s - version %s\n", ixgbevf_driver_string,
4057                 ixgbevf_driver_version);
4058
4059         pr_info("%s\n", ixgbevf_copyright);
4060
4061         ret = pci_register_driver(&ixgbevf_driver);
4062         return ret;
4063 }
4064
4065 module_init(ixgbevf_init_module);
4066
4067 /**
4068  * ixgbevf_exit_module - Driver Exit Cleanup Routine
4069  *
4070  * ixgbevf_exit_module is called just before the driver is removed
4071  * from memory.
4072  **/
4073 static void __exit ixgbevf_exit_module(void)
4074 {
4075         pci_unregister_driver(&ixgbevf_driver);
4076 }
4077
4078 #ifdef DEBUG
4079 /**
4080  * ixgbevf_get_hw_dev_name - return device name string
4081  * used by hardware layer to print debugging information
4082  **/
4083 char *ixgbevf_get_hw_dev_name(struct ixgbe_hw *hw)
4084 {
4085         struct ixgbevf_adapter *adapter = hw->back;
4086         return adapter->netdev->name;
4087 }
4088
4089 #endif
4090 module_exit(ixgbevf_exit_module);
4091
4092 /* ixgbevf_main.c */