Merge git://github.com/Jkirsher/net-next
[linux-drm-fsl-dcu.git] / drivers / net / ethernet / intel / ixgbe / ixgbe_main.c
1 /*******************************************************************************
2
3   Intel 10 Gigabit PCI Express Linux driver
4   Copyright(c) 1999 - 2011 Intel Corporation.
5
6   This program is free software; you can redistribute it and/or modify it
7   under the terms and conditions of the GNU General Public License,
8   version 2, as published by the Free Software Foundation.
9
10   This program is distributed in the hope it will be useful, but WITHOUT
11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
13   more details.
14
15   You should have received a copy of the GNU General Public License along with
16   this program; if not, write to the Free Software Foundation, Inc.,
17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19   The full GNU General Public License is included in this distribution in
20   the file called "COPYING".
21
22   Contact Information:
23   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26 *******************************************************************************/
27
28 #include <linux/types.h>
29 #include <linux/module.h>
30 #include <linux/pci.h>
31 #include <linux/netdevice.h>
32 #include <linux/vmalloc.h>
33 #include <linux/string.h>
34 #include <linux/in.h>
35 #include <linux/interrupt.h>
36 #include <linux/ip.h>
37 #include <linux/tcp.h>
38 #include <linux/sctp.h>
39 #include <linux/pkt_sched.h>
40 #include <linux/ipv6.h>
41 #include <linux/slab.h>
42 #include <net/checksum.h>
43 #include <net/ip6_checksum.h>
44 #include <linux/ethtool.h>
45 #include <linux/if.h>
46 #include <linux/if_vlan.h>
47 #include <linux/prefetch.h>
48 #include <scsi/fc/fc_fcoe.h>
49
50 #include "ixgbe.h"
51 #include "ixgbe_common.h"
52 #include "ixgbe_dcb_82599.h"
53 #include "ixgbe_sriov.h"
54
55 char ixgbe_driver_name[] = "ixgbe";
56 static const char ixgbe_driver_string[] =
57                               "Intel(R) 10 Gigabit PCI Express Network Driver";
58 #define MAJ 3
59 #define MIN 4
60 #define BUILD 8
61 #define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
62         __stringify(BUILD) "-k"
63 const char ixgbe_driver_version[] = DRV_VERSION;
64 static const char ixgbe_copyright[] =
65                                 "Copyright (c) 1999-2011 Intel Corporation.";
66
67 static const struct ixgbe_info *ixgbe_info_tbl[] = {
68         [board_82598] = &ixgbe_82598_info,
69         [board_82599] = &ixgbe_82599_info,
70         [board_X540] = &ixgbe_X540_info,
71 };
72
73 /* ixgbe_pci_tbl - PCI Device ID Table
74  *
75  * Wildcard entries (PCI_ANY_ID) should come last
76  * Last entry must be all 0s
77  *
78  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
79  *   Class, Class Mask, private data (not used) }
80  */
81 static DEFINE_PCI_DEVICE_TABLE(ixgbe_pci_tbl) = {
82         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598), board_82598 },
83         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 },
84         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 },
85         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 },
86         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT2), board_82598 },
87         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 },
88         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 },
89         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 },
90         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 },
91         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 },
92         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 },
93         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_BX), board_82598 },
94         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4), board_82599 },
95         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_XAUI_LOM), board_82599 },
96         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KR), board_82599 },
97         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP), board_82599 },
98         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_EM), board_82599 },
99         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_KX4_MEZZ), board_82599 },
100         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_CX4), board_82599 },
101         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_BACKPLANE_FCOE), board_82599 },
102         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_FCOE), board_82599 },
103         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_T3_LOM), board_82599 },
104         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_COMBO_BACKPLANE), board_82599 },
105         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_X540T), board_X540 },
106         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_SFP_SF2), board_82599 },
107         {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82599_LS), board_82599 },
108         /* required last entry */
109         {0, }
110 };
111 MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl);
112
113 #ifdef CONFIG_IXGBE_DCA
114 static int ixgbe_notify_dca(struct notifier_block *, unsigned long event,
115                             void *p);
116 static struct notifier_block dca_notifier = {
117         .notifier_call = ixgbe_notify_dca,
118         .next          = NULL,
119         .priority      = 0
120 };
121 #endif
122
123 #ifdef CONFIG_PCI_IOV
124 static unsigned int max_vfs;
125 module_param(max_vfs, uint, 0);
126 MODULE_PARM_DESC(max_vfs,
127                  "Maximum number of virtual functions to allocate per physical function");
128 #endif /* CONFIG_PCI_IOV */
129
130 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
131 MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver");
132 MODULE_LICENSE("GPL");
133 MODULE_VERSION(DRV_VERSION);
134
135 #define DEFAULT_DEBUG_LEVEL_SHIFT 3
136
137 static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter)
138 {
139         if (!test_bit(__IXGBE_DOWN, &adapter->state) &&
140             !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state))
141                 schedule_work(&adapter->service_task);
142 }
143
144 static void ixgbe_service_event_complete(struct ixgbe_adapter *adapter)
145 {
146         BUG_ON(!test_bit(__IXGBE_SERVICE_SCHED, &adapter->state));
147
148         /* flush memory to make sure state is correct before next watchog */
149         smp_mb__before_clear_bit();
150         clear_bit(__IXGBE_SERVICE_SCHED, &adapter->state);
151 }
152
153 struct ixgbe_reg_info {
154         u32 ofs;
155         char *name;
156 };
157
158 static const struct ixgbe_reg_info ixgbe_reg_info_tbl[] = {
159
160         /* General Registers */
161         {IXGBE_CTRL, "CTRL"},
162         {IXGBE_STATUS, "STATUS"},
163         {IXGBE_CTRL_EXT, "CTRL_EXT"},
164
165         /* Interrupt Registers */
166         {IXGBE_EICR, "EICR"},
167
168         /* RX Registers */
169         {IXGBE_SRRCTL(0), "SRRCTL"},
170         {IXGBE_DCA_RXCTRL(0), "DRXCTL"},
171         {IXGBE_RDLEN(0), "RDLEN"},
172         {IXGBE_RDH(0), "RDH"},
173         {IXGBE_RDT(0), "RDT"},
174         {IXGBE_RXDCTL(0), "RXDCTL"},
175         {IXGBE_RDBAL(0), "RDBAL"},
176         {IXGBE_RDBAH(0), "RDBAH"},
177
178         /* TX Registers */
179         {IXGBE_TDBAL(0), "TDBAL"},
180         {IXGBE_TDBAH(0), "TDBAH"},
181         {IXGBE_TDLEN(0), "TDLEN"},
182         {IXGBE_TDH(0), "TDH"},
183         {IXGBE_TDT(0), "TDT"},
184         {IXGBE_TXDCTL(0), "TXDCTL"},
185
186         /* List Terminator */
187         {}
188 };
189
190
191 /*
192  * ixgbe_regdump - register printout routine
193  */
194 static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo)
195 {
196         int i = 0, j = 0;
197         char rname[16];
198         u32 regs[64];
199
200         switch (reginfo->ofs) {
201         case IXGBE_SRRCTL(0):
202                 for (i = 0; i < 64; i++)
203                         regs[i] = IXGBE_READ_REG(hw, IXGBE_SRRCTL(i));
204                 break;
205         case IXGBE_DCA_RXCTRL(0):
206                 for (i = 0; i < 64; i++)
207                         regs[i] = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(i));
208                 break;
209         case IXGBE_RDLEN(0):
210                 for (i = 0; i < 64; i++)
211                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDLEN(i));
212                 break;
213         case IXGBE_RDH(0):
214                 for (i = 0; i < 64; i++)
215                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDH(i));
216                 break;
217         case IXGBE_RDT(0):
218                 for (i = 0; i < 64; i++)
219                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDT(i));
220                 break;
221         case IXGBE_RXDCTL(0):
222                 for (i = 0; i < 64; i++)
223                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RXDCTL(i));
224                 break;
225         case IXGBE_RDBAL(0):
226                 for (i = 0; i < 64; i++)
227                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAL(i));
228                 break;
229         case IXGBE_RDBAH(0):
230                 for (i = 0; i < 64; i++)
231                         regs[i] = IXGBE_READ_REG(hw, IXGBE_RDBAH(i));
232                 break;
233         case IXGBE_TDBAL(0):
234                 for (i = 0; i < 64; i++)
235                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAL(i));
236                 break;
237         case IXGBE_TDBAH(0):
238                 for (i = 0; i < 64; i++)
239                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDBAH(i));
240                 break;
241         case IXGBE_TDLEN(0):
242                 for (i = 0; i < 64; i++)
243                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDLEN(i));
244                 break;
245         case IXGBE_TDH(0):
246                 for (i = 0; i < 64; i++)
247                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDH(i));
248                 break;
249         case IXGBE_TDT(0):
250                 for (i = 0; i < 64; i++)
251                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TDT(i));
252                 break;
253         case IXGBE_TXDCTL(0):
254                 for (i = 0; i < 64; i++)
255                         regs[i] = IXGBE_READ_REG(hw, IXGBE_TXDCTL(i));
256                 break;
257         default:
258                 pr_info("%-15s %08x\n", reginfo->name,
259                         IXGBE_READ_REG(hw, reginfo->ofs));
260                 return;
261         }
262
263         for (i = 0; i < 8; i++) {
264                 snprintf(rname, 16, "%s[%d-%d]", reginfo->name, i*8, i*8+7);
265                 pr_err("%-15s", rname);
266                 for (j = 0; j < 8; j++)
267                         pr_cont(" %08x", regs[i*8+j]);
268                 pr_cont("\n");
269         }
270
271 }
272
273 /*
274  * ixgbe_dump - Print registers, tx-rings and rx-rings
275  */
276 static void ixgbe_dump(struct ixgbe_adapter *adapter)
277 {
278         struct net_device *netdev = adapter->netdev;
279         struct ixgbe_hw *hw = &adapter->hw;
280         struct ixgbe_reg_info *reginfo;
281         int n = 0;
282         struct ixgbe_ring *tx_ring;
283         struct ixgbe_tx_buffer *tx_buffer_info;
284         union ixgbe_adv_tx_desc *tx_desc;
285         struct my_u0 { u64 a; u64 b; } *u0;
286         struct ixgbe_ring *rx_ring;
287         union ixgbe_adv_rx_desc *rx_desc;
288         struct ixgbe_rx_buffer *rx_buffer_info;
289         u32 staterr;
290         int i = 0;
291
292         if (!netif_msg_hw(adapter))
293                 return;
294
295         /* Print netdevice Info */
296         if (netdev) {
297                 dev_info(&adapter->pdev->dev, "Net device Info\n");
298                 pr_info("Device Name     state            "
299                         "trans_start      last_rx\n");
300                 pr_info("%-15s %016lX %016lX %016lX\n",
301                         netdev->name,
302                         netdev->state,
303                         netdev->trans_start,
304                         netdev->last_rx);
305         }
306
307         /* Print Registers */
308         dev_info(&adapter->pdev->dev, "Register Dump\n");
309         pr_info(" Register Name   Value\n");
310         for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl;
311              reginfo->name; reginfo++) {
312                 ixgbe_regdump(hw, reginfo);
313         }
314
315         /* Print TX Ring Summary */
316         if (!netdev || !netif_running(netdev))
317                 goto exit;
318
319         dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
320         pr_info("Queue [NTU] [NTC] [bi(ntc)->dma  ] leng ntw timestamp\n");
321         for (n = 0; n < adapter->num_tx_queues; n++) {
322                 tx_ring = adapter->tx_ring[n];
323                 tx_buffer_info =
324                         &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
325                 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
326                            n, tx_ring->next_to_use, tx_ring->next_to_clean,
327                            (u64)tx_buffer_info->dma,
328                            tx_buffer_info->length,
329                            tx_buffer_info->next_to_watch,
330                            (u64)tx_buffer_info->time_stamp);
331         }
332
333         /* Print TX Rings */
334         if (!netif_msg_tx_done(adapter))
335                 goto rx_ring_summary;
336
337         dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
338
339         /* Transmit Descriptor Formats
340          *
341          * Advanced Transmit Descriptor
342          *   +--------------------------------------------------------------+
343          * 0 |         Buffer Address [63:0]                                |
344          *   +--------------------------------------------------------------+
345          * 8 |  PAYLEN  | PORTS  | IDX | STA | DCMD  |DTYP |  RSV |  DTALEN |
346          *   +--------------------------------------------------------------+
347          *   63       46 45    40 39 36 35 32 31   24 23 20 19              0
348          */
349
350         for (n = 0; n < adapter->num_tx_queues; n++) {
351                 tx_ring = adapter->tx_ring[n];
352                 pr_info("------------------------------------\n");
353                 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
354                 pr_info("------------------------------------\n");
355                 pr_info("T [desc]     [address 63:0  ] "
356                         "[PlPOIdStDDt Ln] [bi->dma       ] "
357                         "leng  ntw timestamp        bi->skb\n");
358
359                 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
360                         tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
361                         tx_buffer_info = &tx_ring->tx_buffer_info[i];
362                         u0 = (struct my_u0 *)tx_desc;
363                         pr_info("T [0x%03X]    %016llX %016llX %016llX"
364                                 " %04X  %p %016llX %p", i,
365                                 le64_to_cpu(u0->a),
366                                 le64_to_cpu(u0->b),
367                                 (u64)tx_buffer_info->dma,
368                                 tx_buffer_info->length,
369                                 tx_buffer_info->next_to_watch,
370                                 (u64)tx_buffer_info->time_stamp,
371                                 tx_buffer_info->skb);
372                         if (i == tx_ring->next_to_use &&
373                                 i == tx_ring->next_to_clean)
374                                 pr_cont(" NTC/U\n");
375                         else if (i == tx_ring->next_to_use)
376                                 pr_cont(" NTU\n");
377                         else if (i == tx_ring->next_to_clean)
378                                 pr_cont(" NTC\n");
379                         else
380                                 pr_cont("\n");
381
382                         if (netif_msg_pktdata(adapter) &&
383                                 tx_buffer_info->dma != 0)
384                                 print_hex_dump(KERN_INFO, "",
385                                         DUMP_PREFIX_ADDRESS, 16, 1,
386                                         phys_to_virt(tx_buffer_info->dma),
387                                         tx_buffer_info->length, true);
388                 }
389         }
390
391         /* Print RX Rings Summary */
392 rx_ring_summary:
393         dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
394         pr_info("Queue [NTU] [NTC]\n");
395         for (n = 0; n < adapter->num_rx_queues; n++) {
396                 rx_ring = adapter->rx_ring[n];
397                 pr_info("%5d %5X %5X\n",
398                         n, rx_ring->next_to_use, rx_ring->next_to_clean);
399         }
400
401         /* Print RX Rings */
402         if (!netif_msg_rx_status(adapter))
403                 goto exit;
404
405         dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
406
407         /* Advanced Receive Descriptor (Read) Format
408          *    63                                           1        0
409          *    +-----------------------------------------------------+
410          *  0 |       Packet Buffer Address [63:1]           |A0/NSE|
411          *    +----------------------------------------------+------+
412          *  8 |       Header Buffer Address [63:1]           |  DD  |
413          *    +-----------------------------------------------------+
414          *
415          *
416          * Advanced Receive Descriptor (Write-Back) Format
417          *
418          *   63       48 47    32 31  30      21 20 16 15   4 3     0
419          *   +------------------------------------------------------+
420          * 0 | Packet     IP     |SPH| HDR_LEN   | RSV|Packet|  RSS |
421          *   | Checksum   Ident  |   |           |    | Type | Type |
422          *   +------------------------------------------------------+
423          * 8 | VLAN Tag | Length | Extended Error | Extended Status |
424          *   +------------------------------------------------------+
425          *   63       48 47    32 31            20 19               0
426          */
427         for (n = 0; n < adapter->num_rx_queues; n++) {
428                 rx_ring = adapter->rx_ring[n];
429                 pr_info("------------------------------------\n");
430                 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
431                 pr_info("------------------------------------\n");
432                 pr_info("R  [desc]      [ PktBuf     A0] "
433                         "[  HeadBuf   DD] [bi->dma       ] [bi->skb] "
434                         "<-- Adv Rx Read format\n");
435                 pr_info("RWB[desc]      [PcsmIpSHl PtRs] "
436                         "[vl er S cks ln] ---------------- [bi->skb] "
437                         "<-- Adv Rx Write-Back format\n");
438
439                 for (i = 0; i < rx_ring->count; i++) {
440                         rx_buffer_info = &rx_ring->rx_buffer_info[i];
441                         rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
442                         u0 = (struct my_u0 *)rx_desc;
443                         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
444                         if (staterr & IXGBE_RXD_STAT_DD) {
445                                 /* Descriptor Done */
446                                 pr_info("RWB[0x%03X]     %016llX "
447                                         "%016llX ---------------- %p", i,
448                                         le64_to_cpu(u0->a),
449                                         le64_to_cpu(u0->b),
450                                         rx_buffer_info->skb);
451                         } else {
452                                 pr_info("R  [0x%03X]     %016llX "
453                                         "%016llX %016llX %p", i,
454                                         le64_to_cpu(u0->a),
455                                         le64_to_cpu(u0->b),
456                                         (u64)rx_buffer_info->dma,
457                                         rx_buffer_info->skb);
458
459                                 if (netif_msg_pktdata(adapter)) {
460                                         print_hex_dump(KERN_INFO, "",
461                                            DUMP_PREFIX_ADDRESS, 16, 1,
462                                            phys_to_virt(rx_buffer_info->dma),
463                                            rx_ring->rx_buf_len, true);
464
465                                         if (rx_ring->rx_buf_len
466                                                 < IXGBE_RXBUFFER_2K)
467                                                 print_hex_dump(KERN_INFO, "",
468                                                   DUMP_PREFIX_ADDRESS, 16, 1,
469                                                   phys_to_virt(
470                                                     rx_buffer_info->page_dma +
471                                                     rx_buffer_info->page_offset
472                                                   ),
473                                                   PAGE_SIZE/2, true);
474                                 }
475                         }
476
477                         if (i == rx_ring->next_to_use)
478                                 pr_cont(" NTU\n");
479                         else if (i == rx_ring->next_to_clean)
480                                 pr_cont(" NTC\n");
481                         else
482                                 pr_cont("\n");
483
484                 }
485         }
486
487 exit:
488         return;
489 }
490
491 static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter)
492 {
493         u32 ctrl_ext;
494
495         /* Let firmware take over control of h/w */
496         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
497         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
498                         ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD);
499 }
500
501 static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter)
502 {
503         u32 ctrl_ext;
504
505         /* Let firmware know the driver has taken over */
506         ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT);
507         IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT,
508                         ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD);
509 }
510
511 /*
512  * ixgbe_set_ivar - set the IVAR registers, mapping interrupt causes to vectors
513  * @adapter: pointer to adapter struct
514  * @direction: 0 for Rx, 1 for Tx, -1 for other causes
515  * @queue: queue to map the corresponding interrupt to
516  * @msix_vector: the vector to map to the corresponding queue
517  *
518  */
519 static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction,
520                            u8 queue, u8 msix_vector)
521 {
522         u32 ivar, index;
523         struct ixgbe_hw *hw = &adapter->hw;
524         switch (hw->mac.type) {
525         case ixgbe_mac_82598EB:
526                 msix_vector |= IXGBE_IVAR_ALLOC_VAL;
527                 if (direction == -1)
528                         direction = 0;
529                 index = (((direction * 64) + queue) >> 2) & 0x1F;
530                 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index));
531                 ivar &= ~(0xFF << (8 * (queue & 0x3)));
532                 ivar |= (msix_vector << (8 * (queue & 0x3)));
533                 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar);
534                 break;
535         case ixgbe_mac_82599EB:
536         case ixgbe_mac_X540:
537                 if (direction == -1) {
538                         /* other causes */
539                         msix_vector |= IXGBE_IVAR_ALLOC_VAL;
540                         index = ((queue & 1) * 8);
541                         ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR_MISC);
542                         ivar &= ~(0xFF << index);
543                         ivar |= (msix_vector << index);
544                         IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR_MISC, ivar);
545                         break;
546                 } else {
547                         /* tx or rx causes */
548                         msix_vector |= IXGBE_IVAR_ALLOC_VAL;
549                         index = ((16 * (queue & 1)) + (8 * direction));
550                         ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(queue >> 1));
551                         ivar &= ~(0xFF << index);
552                         ivar |= (msix_vector << index);
553                         IXGBE_WRITE_REG(hw, IXGBE_IVAR(queue >> 1), ivar);
554                         break;
555                 }
556         default:
557                 break;
558         }
559 }
560
561 static inline void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter,
562                                           u64 qmask)
563 {
564         u32 mask;
565
566         switch (adapter->hw.mac.type) {
567         case ixgbe_mac_82598EB:
568                 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
569                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, mask);
570                 break;
571         case ixgbe_mac_82599EB:
572         case ixgbe_mac_X540:
573                 mask = (qmask & 0xFFFFFFFF);
574                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(0), mask);
575                 mask = (qmask >> 32);
576                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS_EX(1), mask);
577                 break;
578         default:
579                 break;
580         }
581 }
582
583 static inline void ixgbe_unmap_tx_resource(struct ixgbe_ring *ring,
584                                            struct ixgbe_tx_buffer *tx_buffer)
585 {
586         if (tx_buffer->dma) {
587                 if (tx_buffer->tx_flags & IXGBE_TX_FLAGS_MAPPED_AS_PAGE)
588                         dma_unmap_page(ring->dev,
589                                        tx_buffer->dma,
590                                        tx_buffer->length,
591                                        DMA_TO_DEVICE);
592                 else
593                         dma_unmap_single(ring->dev,
594                                          tx_buffer->dma,
595                                          tx_buffer->length,
596                                          DMA_TO_DEVICE);
597         }
598         tx_buffer->dma = 0;
599 }
600
601 void ixgbe_unmap_and_free_tx_resource(struct ixgbe_ring *tx_ring,
602                                       struct ixgbe_tx_buffer *tx_buffer_info)
603 {
604         ixgbe_unmap_tx_resource(tx_ring, tx_buffer_info);
605         if (tx_buffer_info->skb)
606                 dev_kfree_skb_any(tx_buffer_info->skb);
607         tx_buffer_info->skb = NULL;
608         /* tx_buffer_info must be completely set up in the transmit path */
609 }
610
611 static void ixgbe_update_xoff_received(struct ixgbe_adapter *adapter)
612 {
613         struct ixgbe_hw *hw = &adapter->hw;
614         struct ixgbe_hw_stats *hwstats = &adapter->stats;
615         u32 data = 0;
616         u32 xoff[8] = {0};
617         int i;
618
619         if ((hw->fc.current_mode == ixgbe_fc_full) ||
620             (hw->fc.current_mode == ixgbe_fc_rx_pause)) {
621                 switch (hw->mac.type) {
622                 case ixgbe_mac_82598EB:
623                         data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXC);
624                         break;
625                 default:
626                         data = IXGBE_READ_REG(hw, IXGBE_LXOFFRXCNT);
627                 }
628                 hwstats->lxoffrxc += data;
629
630                 /* refill credits (no tx hang) if we received xoff */
631                 if (!data)
632                         return;
633
634                 for (i = 0; i < adapter->num_tx_queues; i++)
635                         clear_bit(__IXGBE_HANG_CHECK_ARMED,
636                                   &adapter->tx_ring[i]->state);
637                 return;
638         } else if (!(adapter->dcb_cfg.pfc_mode_enable))
639                 return;
640
641         /* update stats for each tc, only valid with PFC enabled */
642         for (i = 0; i < MAX_TX_PACKET_BUFFERS; i++) {
643                 switch (hw->mac.type) {
644                 case ixgbe_mac_82598EB:
645                         xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXC(i));
646                         break;
647                 default:
648                         xoff[i] = IXGBE_READ_REG(hw, IXGBE_PXOFFRXCNT(i));
649                 }
650                 hwstats->pxoffrxc[i] += xoff[i];
651         }
652
653         /* disarm tx queues that have received xoff frames */
654         for (i = 0; i < adapter->num_tx_queues; i++) {
655                 struct ixgbe_ring *tx_ring = adapter->tx_ring[i];
656                 u8 tc = tx_ring->dcb_tc;
657
658                 if (xoff[tc])
659                         clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
660         }
661 }
662
663 static u64 ixgbe_get_tx_completed(struct ixgbe_ring *ring)
664 {
665         return ring->tx_stats.completed;
666 }
667
668 static u64 ixgbe_get_tx_pending(struct ixgbe_ring *ring)
669 {
670         struct ixgbe_adapter *adapter = netdev_priv(ring->netdev);
671         struct ixgbe_hw *hw = &adapter->hw;
672
673         u32 head = IXGBE_READ_REG(hw, IXGBE_TDH(ring->reg_idx));
674         u32 tail = IXGBE_READ_REG(hw, IXGBE_TDT(ring->reg_idx));
675
676         if (head != tail)
677                 return (head < tail) ?
678                         tail - head : (tail + ring->count - head);
679
680         return 0;
681 }
682
683 static inline bool ixgbe_check_tx_hang(struct ixgbe_ring *tx_ring)
684 {
685         u32 tx_done = ixgbe_get_tx_completed(tx_ring);
686         u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
687         u32 tx_pending = ixgbe_get_tx_pending(tx_ring);
688         bool ret = false;
689
690         clear_check_for_tx_hang(tx_ring);
691
692         /*
693          * Check for a hung queue, but be thorough. This verifies
694          * that a transmit has been completed since the previous
695          * check AND there is at least one packet pending. The
696          * ARMED bit is set to indicate a potential hang. The
697          * bit is cleared if a pause frame is received to remove
698          * false hang detection due to PFC or 802.3x frames. By
699          * requiring this to fail twice we avoid races with
700          * pfc clearing the ARMED bit and conditions where we
701          * run the check_tx_hang logic with a transmit completion
702          * pending but without time to complete it yet.
703          */
704         if ((tx_done_old == tx_done) && tx_pending) {
705                 /* make sure it is true for two checks in a row */
706                 ret = test_and_set_bit(__IXGBE_HANG_CHECK_ARMED,
707                                        &tx_ring->state);
708         } else {
709                 /* update completed stats and continue */
710                 tx_ring->tx_stats.tx_done_old = tx_done;
711                 /* reset the countdown */
712                 clear_bit(__IXGBE_HANG_CHECK_ARMED, &tx_ring->state);
713         }
714
715         return ret;
716 }
717
718 /**
719  * ixgbe_tx_timeout_reset - initiate reset due to Tx timeout
720  * @adapter: driver private struct
721  **/
722 static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter)
723 {
724
725         /* Do the reset outside of interrupt context */
726         if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
727                 adapter->flags2 |= IXGBE_FLAG2_RESET_REQUESTED;
728                 ixgbe_service_event_schedule(adapter);
729         }
730 }
731
732 /**
733  * ixgbe_clean_tx_irq - Reclaim resources after transmit completes
734  * @q_vector: structure containing interrupt and ring information
735  * @tx_ring: tx ring to clean
736  **/
737 static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector,
738                                struct ixgbe_ring *tx_ring)
739 {
740         struct ixgbe_adapter *adapter = q_vector->adapter;
741         struct ixgbe_tx_buffer *tx_buffer;
742         union ixgbe_adv_tx_desc *tx_desc;
743         unsigned int total_bytes = 0, total_packets = 0;
744         unsigned int budget = q_vector->tx.work_limit;
745         u16 i = tx_ring->next_to_clean;
746
747         tx_buffer = &tx_ring->tx_buffer_info[i];
748         tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
749
750         for (; budget; budget--) {
751                 union ixgbe_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
752
753                 /* if next_to_watch is not set then there is no work pending */
754                 if (!eop_desc)
755                         break;
756
757                 /* if DD is not set pending work has not been completed */
758                 if (!(eop_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD)))
759                         break;
760
761                 /* count the packet as being completed */
762                 tx_ring->tx_stats.completed++;
763
764                 /* clear next_to_watch to prevent false hangs */
765                 tx_buffer->next_to_watch = NULL;
766
767                 /* prevent any other reads prior to eop_desc being verified */
768                 rmb();
769
770                 do {
771                         ixgbe_unmap_tx_resource(tx_ring, tx_buffer);
772                         tx_desc->wb.status = 0;
773                         if (likely(tx_desc == eop_desc)) {
774                                 eop_desc = NULL;
775                                 dev_kfree_skb_any(tx_buffer->skb);
776                                 tx_buffer->skb = NULL;
777
778                                 total_bytes += tx_buffer->bytecount;
779                                 total_packets += tx_buffer->gso_segs;
780                         }
781
782                         tx_buffer++;
783                         tx_desc++;
784                         i++;
785                         if (unlikely(i == tx_ring->count)) {
786                                 i = 0;
787
788                                 tx_buffer = tx_ring->tx_buffer_info;
789                                 tx_desc = IXGBE_TX_DESC_ADV(tx_ring, 0);
790                         }
791
792                 } while (eop_desc);
793         }
794
795         tx_ring->next_to_clean = i;
796         u64_stats_update_begin(&tx_ring->syncp);
797         tx_ring->stats.bytes += total_bytes;
798         tx_ring->stats.packets += total_packets;
799         u64_stats_update_end(&tx_ring->syncp);
800         q_vector->tx.total_bytes += total_bytes;
801         q_vector->tx.total_packets += total_packets;
802
803         if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) {
804                 /* schedule immediate reset if we believe we hung */
805                 struct ixgbe_hw *hw = &adapter->hw;
806                 tx_desc = IXGBE_TX_DESC_ADV(tx_ring, i);
807                 e_err(drv, "Detected Tx Unit Hang\n"
808                         "  Tx Queue             <%d>\n"
809                         "  TDH, TDT             <%x>, <%x>\n"
810                         "  next_to_use          <%x>\n"
811                         "  next_to_clean        <%x>\n"
812                         "tx_buffer_info[next_to_clean]\n"
813                         "  time_stamp           <%lx>\n"
814                         "  jiffies              <%lx>\n",
815                         tx_ring->queue_index,
816                         IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)),
817                         IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)),
818                         tx_ring->next_to_use, i,
819                         tx_ring->tx_buffer_info[i].time_stamp, jiffies);
820
821                 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
822
823                 e_info(probe,
824                        "tx hang %d detected on queue %d, resetting adapter\n",
825                         adapter->tx_timeout_count + 1, tx_ring->queue_index);
826
827                 /* schedule immediate reset if we believe we hung */
828                 ixgbe_tx_timeout_reset(adapter);
829
830                 /* the adapter is about to reset, no point in enabling stuff */
831                 return true;
832         }
833
834 #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
835         if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
836                      (ixgbe_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
837                 /* Make sure that anybody stopping the queue after this
838                  * sees the new next_to_clean.
839                  */
840                 smp_mb();
841                 if (__netif_subqueue_stopped(tx_ring->netdev, tx_ring->queue_index) &&
842                     !test_bit(__IXGBE_DOWN, &adapter->state)) {
843                         netif_wake_subqueue(tx_ring->netdev, tx_ring->queue_index);
844                         ++tx_ring->tx_stats.restart_queue;
845                 }
846         }
847
848         return !!budget;
849 }
850
851 #ifdef CONFIG_IXGBE_DCA
852 static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter,
853                                 struct ixgbe_ring *rx_ring,
854                                 int cpu)
855 {
856         struct ixgbe_hw *hw = &adapter->hw;
857         u32 rxctrl;
858         u8 reg_idx = rx_ring->reg_idx;
859
860         rxctrl = IXGBE_READ_REG(hw, IXGBE_DCA_RXCTRL(reg_idx));
861         switch (hw->mac.type) {
862         case ixgbe_mac_82598EB:
863                 rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK;
864                 rxctrl |= dca3_get_tag(rx_ring->dev, cpu);
865                 break;
866         case ixgbe_mac_82599EB:
867         case ixgbe_mac_X540:
868                 rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK_82599;
869                 rxctrl |= (dca3_get_tag(rx_ring->dev, cpu) <<
870                            IXGBE_DCA_RXCTRL_CPUID_SHIFT_82599);
871                 break;
872         default:
873                 break;
874         }
875         rxctrl |= IXGBE_DCA_RXCTRL_DESC_DCA_EN;
876         rxctrl |= IXGBE_DCA_RXCTRL_HEAD_DCA_EN;
877         rxctrl &= ~(IXGBE_DCA_RXCTRL_DESC_RRO_EN);
878         IXGBE_WRITE_REG(hw, IXGBE_DCA_RXCTRL(reg_idx), rxctrl);
879 }
880
881 static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter,
882                                 struct ixgbe_ring *tx_ring,
883                                 int cpu)
884 {
885         struct ixgbe_hw *hw = &adapter->hw;
886         u32 txctrl;
887         u8 reg_idx = tx_ring->reg_idx;
888
889         switch (hw->mac.type) {
890         case ixgbe_mac_82598EB:
891                 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(reg_idx));
892                 txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK;
893                 txctrl |= dca3_get_tag(tx_ring->dev, cpu);
894                 txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN;
895                 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL(reg_idx), txctrl);
896                 break;
897         case ixgbe_mac_82599EB:
898         case ixgbe_mac_X540:
899                 txctrl = IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx));
900                 txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK_82599;
901                 txctrl |= (dca3_get_tag(tx_ring->dev, cpu) <<
902                            IXGBE_DCA_TXCTRL_CPUID_SHIFT_82599);
903                 txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN;
904                 IXGBE_WRITE_REG(hw, IXGBE_DCA_TXCTRL_82599(reg_idx), txctrl);
905                 break;
906         default:
907                 break;
908         }
909 }
910
911 static void ixgbe_update_dca(struct ixgbe_q_vector *q_vector)
912 {
913         struct ixgbe_adapter *adapter = q_vector->adapter;
914         struct ixgbe_ring *ring;
915         int cpu = get_cpu();
916
917         if (q_vector->cpu == cpu)
918                 goto out_no_update;
919
920         for (ring = q_vector->tx.ring; ring != NULL; ring = ring->next)
921                 ixgbe_update_tx_dca(adapter, ring, cpu);
922
923         for (ring = q_vector->rx.ring; ring != NULL; ring = ring->next)
924                 ixgbe_update_rx_dca(adapter, ring, cpu);
925
926         q_vector->cpu = cpu;
927 out_no_update:
928         put_cpu();
929 }
930
931 static void ixgbe_setup_dca(struct ixgbe_adapter *adapter)
932 {
933         int num_q_vectors;
934         int i;
935
936         if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED))
937                 return;
938
939         /* always use CB2 mode, difference is masked in the CB driver */
940         IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 2);
941
942         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
943                 num_q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
944         else
945                 num_q_vectors = 1;
946
947         for (i = 0; i < num_q_vectors; i++) {
948                 adapter->q_vector[i]->cpu = -1;
949                 ixgbe_update_dca(adapter->q_vector[i]);
950         }
951 }
952
953 static int __ixgbe_notify_dca(struct device *dev, void *data)
954 {
955         struct ixgbe_adapter *adapter = dev_get_drvdata(dev);
956         unsigned long event = *(unsigned long *)data;
957
958         if (!(adapter->flags & IXGBE_FLAG_DCA_CAPABLE))
959                 return 0;
960
961         switch (event) {
962         case DCA_PROVIDER_ADD:
963                 /* if we're already enabled, don't do it again */
964                 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED)
965                         break;
966                 if (dca_add_requester(dev) == 0) {
967                         adapter->flags |= IXGBE_FLAG_DCA_ENABLED;
968                         ixgbe_setup_dca(adapter);
969                         break;
970                 }
971                 /* Fall Through since DCA is disabled. */
972         case DCA_PROVIDER_REMOVE:
973                 if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) {
974                         dca_remove_requester(dev);
975                         adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED;
976                         IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1);
977                 }
978                 break;
979         }
980
981         return 0;
982 }
983 #endif /* CONFIG_IXGBE_DCA */
984
985 static inline void ixgbe_rx_hash(union ixgbe_adv_rx_desc *rx_desc,
986                                  struct sk_buff *skb)
987 {
988         skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
989 }
990
991 /**
992  * ixgbe_rx_is_fcoe - check the rx desc for incoming pkt type
993  * @adapter: address of board private structure
994  * @rx_desc: advanced rx descriptor
995  *
996  * Returns : true if it is FCoE pkt
997  */
998 static inline bool ixgbe_rx_is_fcoe(struct ixgbe_adapter *adapter,
999                                     union ixgbe_adv_rx_desc *rx_desc)
1000 {
1001         __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1002
1003         return (adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
1004                ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) ==
1005                 (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE <<
1006                              IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT)));
1007 }
1008
1009 /**
1010  * ixgbe_receive_skb - Send a completed packet up the stack
1011  * @adapter: board private structure
1012  * @skb: packet to send up
1013  * @status: hardware indication of status of receive
1014  * @rx_ring: rx descriptor ring (for a specific queue) to setup
1015  * @rx_desc: rx descriptor
1016  **/
1017 static void ixgbe_receive_skb(struct ixgbe_q_vector *q_vector,
1018                               struct sk_buff *skb, u8 status,
1019                               struct ixgbe_ring *ring,
1020                               union ixgbe_adv_rx_desc *rx_desc)
1021 {
1022         struct ixgbe_adapter *adapter = q_vector->adapter;
1023         struct napi_struct *napi = &q_vector->napi;
1024         bool is_vlan = (status & IXGBE_RXD_STAT_VP);
1025         u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan);
1026
1027         if (is_vlan && (tag & VLAN_VID_MASK))
1028                 __vlan_hwaccel_put_tag(skb, tag);
1029
1030         if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL))
1031                 napi_gro_receive(napi, skb);
1032         else
1033                 netif_rx(skb);
1034 }
1035
1036 /**
1037  * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum
1038  * @adapter: address of board private structure
1039  * @status_err: hardware indication of status of receive
1040  * @skb: skb currently being received and modified
1041  * @status_err: status error value of last descriptor in packet
1042  **/
1043 static inline void ixgbe_rx_checksum(struct ixgbe_adapter *adapter,
1044                                      union ixgbe_adv_rx_desc *rx_desc,
1045                                      struct sk_buff *skb,
1046                                      u32 status_err)
1047 {
1048         skb->ip_summed = CHECKSUM_NONE;
1049
1050         /* Rx csum disabled */
1051         if (!(adapter->flags & IXGBE_FLAG_RX_CSUM_ENABLED))
1052                 return;
1053
1054         /* if IP and error */
1055         if ((status_err & IXGBE_RXD_STAT_IPCS) &&
1056             (status_err & IXGBE_RXDADV_ERR_IPE)) {
1057                 adapter->hw_csum_rx_error++;
1058                 return;
1059         }
1060
1061         if (!(status_err & IXGBE_RXD_STAT_L4CS))
1062                 return;
1063
1064         if (status_err & IXGBE_RXDADV_ERR_TCPE) {
1065                 u16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info;
1066
1067                 /*
1068                  * 82599 errata, UDP frames with a 0 checksum can be marked as
1069                  * checksum errors.
1070                  */
1071                 if ((pkt_info & IXGBE_RXDADV_PKTTYPE_UDP) &&
1072                     (adapter->hw.mac.type == ixgbe_mac_82599EB))
1073                         return;
1074
1075                 adapter->hw_csum_rx_error++;
1076                 return;
1077         }
1078
1079         /* It must be a TCP or UDP packet with a valid checksum */
1080         skb->ip_summed = CHECKSUM_UNNECESSARY;
1081 }
1082
1083 static inline void ixgbe_release_rx_desc(struct ixgbe_ring *rx_ring, u32 val)
1084 {
1085         /*
1086          * Force memory writes to complete before letting h/w
1087          * know there are new descriptors to fetch.  (Only
1088          * applicable for weak-ordered memory model archs,
1089          * such as IA-64).
1090          */
1091         wmb();
1092         writel(val, rx_ring->tail);
1093 }
1094
1095 /**
1096  * ixgbe_alloc_rx_buffers - Replace used receive buffers; packet split
1097  * @rx_ring: ring to place buffers on
1098  * @cleaned_count: number of buffers to replace
1099  **/
1100 void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count)
1101 {
1102         union ixgbe_adv_rx_desc *rx_desc;
1103         struct ixgbe_rx_buffer *bi;
1104         struct sk_buff *skb;
1105         u16 i = rx_ring->next_to_use;
1106
1107         /* do nothing if no valid netdev defined */
1108         if (!rx_ring->netdev)
1109                 return;
1110
1111         while (cleaned_count--) {
1112                 rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
1113                 bi = &rx_ring->rx_buffer_info[i];
1114                 skb = bi->skb;
1115
1116                 if (!skb) {
1117                         skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
1118                                                         rx_ring->rx_buf_len);
1119                         if (!skb) {
1120                                 rx_ring->rx_stats.alloc_rx_buff_failed++;
1121                                 goto no_buffers;
1122                         }
1123                         /* initialize queue mapping */
1124                         skb_record_rx_queue(skb, rx_ring->queue_index);
1125                         bi->skb = skb;
1126                 }
1127
1128                 if (!bi->dma) {
1129                         bi->dma = dma_map_single(rx_ring->dev,
1130                                                  skb->data,
1131                                                  rx_ring->rx_buf_len,
1132                                                  DMA_FROM_DEVICE);
1133                         if (dma_mapping_error(rx_ring->dev, bi->dma)) {
1134                                 rx_ring->rx_stats.alloc_rx_buff_failed++;
1135                                 bi->dma = 0;
1136                                 goto no_buffers;
1137                         }
1138                 }
1139
1140                 if (ring_is_ps_enabled(rx_ring)) {
1141                         if (!bi->page) {
1142                                 bi->page = netdev_alloc_page(rx_ring->netdev);
1143                                 if (!bi->page) {
1144                                         rx_ring->rx_stats.alloc_rx_page_failed++;
1145                                         goto no_buffers;
1146                                 }
1147                         }
1148
1149                         if (!bi->page_dma) {
1150                                 /* use a half page if we're re-using */
1151                                 bi->page_offset ^= PAGE_SIZE / 2;
1152                                 bi->page_dma = dma_map_page(rx_ring->dev,
1153                                                             bi->page,
1154                                                             bi->page_offset,
1155                                                             PAGE_SIZE / 2,
1156                                                             DMA_FROM_DEVICE);
1157                                 if (dma_mapping_error(rx_ring->dev,
1158                                                       bi->page_dma)) {
1159                                         rx_ring->rx_stats.alloc_rx_page_failed++;
1160                                         bi->page_dma = 0;
1161                                         goto no_buffers;
1162                                 }
1163                         }
1164
1165                         /* Refresh the desc even if buffer_addrs didn't change
1166                          * because each write-back erases this info. */
1167                         rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
1168                         rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
1169                 } else {
1170                         rx_desc->read.pkt_addr = cpu_to_le64(bi->dma);
1171                         rx_desc->read.hdr_addr = 0;
1172                 }
1173
1174                 i++;
1175                 if (i == rx_ring->count)
1176                         i = 0;
1177         }
1178
1179 no_buffers:
1180         if (rx_ring->next_to_use != i) {
1181                 rx_ring->next_to_use = i;
1182                 ixgbe_release_rx_desc(rx_ring, i);
1183         }
1184 }
1185
1186 static inline u16 ixgbe_get_hlen(union ixgbe_adv_rx_desc *rx_desc)
1187 {
1188         /* HW will not DMA in data larger than the given buffer, even if it
1189          * parses the (NFS, of course) header to be larger.  In that case, it
1190          * fills the header buffer and spills the rest into the page.
1191          */
1192         u16 hdr_info = le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.hdr_info);
1193         u16 hlen = (hdr_info &  IXGBE_RXDADV_HDRBUFLEN_MASK) >>
1194                     IXGBE_RXDADV_HDRBUFLEN_SHIFT;
1195         if (hlen > IXGBE_RX_HDR_SIZE)
1196                 hlen = IXGBE_RX_HDR_SIZE;
1197         return hlen;
1198 }
1199
1200 /**
1201  * ixgbe_transform_rsc_queue - change rsc queue into a full packet
1202  * @skb: pointer to the last skb in the rsc queue
1203  *
1204  * This function changes a queue full of hw rsc buffers into a completed
1205  * packet.  It uses the ->prev pointers to find the first packet and then
1206  * turns it into the frag list owner.
1207  **/
1208 static inline struct sk_buff *ixgbe_transform_rsc_queue(struct sk_buff *skb)
1209 {
1210         unsigned int frag_list_size = 0;
1211         unsigned int skb_cnt = 1;
1212
1213         while (skb->prev) {
1214                 struct sk_buff *prev = skb->prev;
1215                 frag_list_size += skb->len;
1216                 skb->prev = NULL;
1217                 skb = prev;
1218                 skb_cnt++;
1219         }
1220
1221         skb_shinfo(skb)->frag_list = skb->next;
1222         skb->next = NULL;
1223         skb->len += frag_list_size;
1224         skb->data_len += frag_list_size;
1225         skb->truesize += frag_list_size;
1226         IXGBE_RSC_CB(skb)->skb_cnt = skb_cnt;
1227
1228         return skb;
1229 }
1230
1231 static inline bool ixgbe_get_rsc_state(union ixgbe_adv_rx_desc *rx_desc)
1232 {
1233         return !!(le32_to_cpu(rx_desc->wb.lower.lo_dword.data) &
1234                 IXGBE_RXDADV_RSCCNT_MASK);
1235 }
1236
1237 static bool ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector,
1238                                struct ixgbe_ring *rx_ring,
1239                                int budget)
1240 {
1241         struct ixgbe_adapter *adapter = q_vector->adapter;
1242         union ixgbe_adv_rx_desc *rx_desc, *next_rxd;
1243         struct ixgbe_rx_buffer *rx_buffer_info, *next_buffer;
1244         struct sk_buff *skb;
1245         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1246         const int current_node = numa_node_id();
1247 #ifdef IXGBE_FCOE
1248         int ddp_bytes = 0;
1249 #endif /* IXGBE_FCOE */
1250         u32 staterr;
1251         u16 i;
1252         u16 cleaned_count = 0;
1253         bool pkt_is_rsc = false;
1254
1255         i = rx_ring->next_to_clean;
1256         rx_desc = IXGBE_RX_DESC_ADV(rx_ring, i);
1257         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1258
1259         while (staterr & IXGBE_RXD_STAT_DD) {
1260                 u32 upper_len = 0;
1261
1262                 rmb(); /* read descriptor and rx_buffer_info after status DD */
1263
1264                 rx_buffer_info = &rx_ring->rx_buffer_info[i];
1265
1266                 skb = rx_buffer_info->skb;
1267                 rx_buffer_info->skb = NULL;
1268                 prefetch(skb->data);
1269
1270                 if (ring_is_rsc_enabled(rx_ring))
1271                         pkt_is_rsc = ixgbe_get_rsc_state(rx_desc);
1272
1273                 /* linear means we are building an skb from multiple pages */
1274                 if (!skb_is_nonlinear(skb)) {
1275                         u16 hlen;
1276                         if (pkt_is_rsc &&
1277                             !(staterr & IXGBE_RXD_STAT_EOP) &&
1278                             !skb->prev) {
1279                                 /*
1280                                  * When HWRSC is enabled, delay unmapping
1281                                  * of the first packet. It carries the
1282                                  * header information, HW may still
1283                                  * access the header after the writeback.
1284                                  * Only unmap it when EOP is reached
1285                                  */
1286                                 IXGBE_RSC_CB(skb)->delay_unmap = true;
1287                                 IXGBE_RSC_CB(skb)->dma = rx_buffer_info->dma;
1288                         } else {
1289                                 dma_unmap_single(rx_ring->dev,
1290                                                  rx_buffer_info->dma,
1291                                                  rx_ring->rx_buf_len,
1292                                                  DMA_FROM_DEVICE);
1293                         }
1294                         rx_buffer_info->dma = 0;
1295
1296                         if (ring_is_ps_enabled(rx_ring)) {
1297                                 hlen = ixgbe_get_hlen(rx_desc);
1298                                 upper_len = le16_to_cpu(rx_desc->wb.upper.length);
1299                         } else {
1300                                 hlen = le16_to_cpu(rx_desc->wb.upper.length);
1301                         }
1302
1303                         skb_put(skb, hlen);
1304                 } else {
1305                         /* assume packet split since header is unmapped */
1306                         upper_len = le16_to_cpu(rx_desc->wb.upper.length);
1307                 }
1308
1309                 if (upper_len) {
1310                         dma_unmap_page(rx_ring->dev,
1311                                        rx_buffer_info->page_dma,
1312                                        PAGE_SIZE / 2,
1313                                        DMA_FROM_DEVICE);
1314                         rx_buffer_info->page_dma = 0;
1315                         skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
1316                                            rx_buffer_info->page,
1317                                            rx_buffer_info->page_offset,
1318                                            upper_len);
1319
1320                         if ((page_count(rx_buffer_info->page) == 1) &&
1321                             (page_to_nid(rx_buffer_info->page) == current_node))
1322                                 get_page(rx_buffer_info->page);
1323                         else
1324                                 rx_buffer_info->page = NULL;
1325
1326                         skb->len += upper_len;
1327                         skb->data_len += upper_len;
1328                         skb->truesize += upper_len;
1329                 }
1330
1331                 i++;
1332                 if (i == rx_ring->count)
1333                         i = 0;
1334
1335                 next_rxd = IXGBE_RX_DESC_ADV(rx_ring, i);
1336                 prefetch(next_rxd);
1337                 cleaned_count++;
1338
1339                 if (pkt_is_rsc) {
1340                         u32 nextp = (staterr & IXGBE_RXDADV_NEXTP_MASK) >>
1341                                      IXGBE_RXDADV_NEXTP_SHIFT;
1342                         next_buffer = &rx_ring->rx_buffer_info[nextp];
1343                 } else {
1344                         next_buffer = &rx_ring->rx_buffer_info[i];
1345                 }
1346
1347                 if (!(staterr & IXGBE_RXD_STAT_EOP)) {
1348                         if (ring_is_ps_enabled(rx_ring)) {
1349                                 rx_buffer_info->skb = next_buffer->skb;
1350                                 rx_buffer_info->dma = next_buffer->dma;
1351                                 next_buffer->skb = skb;
1352                                 next_buffer->dma = 0;
1353                         } else {
1354                                 skb->next = next_buffer->skb;
1355                                 skb->next->prev = skb;
1356                         }
1357                         rx_ring->rx_stats.non_eop_descs++;
1358                         goto next_desc;
1359                 }
1360
1361                 if (skb->prev) {
1362                         skb = ixgbe_transform_rsc_queue(skb);
1363                         /* if we got here without RSC the packet is invalid */
1364                         if (!pkt_is_rsc) {
1365                                 __pskb_trim(skb, 0);
1366                                 rx_buffer_info->skb = skb;
1367                                 goto next_desc;
1368                         }
1369                 }
1370
1371                 if (ring_is_rsc_enabled(rx_ring)) {
1372                         if (IXGBE_RSC_CB(skb)->delay_unmap) {
1373                                 dma_unmap_single(rx_ring->dev,
1374                                                  IXGBE_RSC_CB(skb)->dma,
1375                                                  rx_ring->rx_buf_len,
1376                                                  DMA_FROM_DEVICE);
1377                                 IXGBE_RSC_CB(skb)->dma = 0;
1378                                 IXGBE_RSC_CB(skb)->delay_unmap = false;
1379                         }
1380                 }
1381                 if (pkt_is_rsc) {
1382                         if (ring_is_ps_enabled(rx_ring))
1383                                 rx_ring->rx_stats.rsc_count +=
1384                                         skb_shinfo(skb)->nr_frags;
1385                         else
1386                                 rx_ring->rx_stats.rsc_count +=
1387                                         IXGBE_RSC_CB(skb)->skb_cnt;
1388                         rx_ring->rx_stats.rsc_flush++;
1389                 }
1390
1391                 /* ERR_MASK will only have valid bits if EOP set */
1392                 if (unlikely(staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK)) {
1393                         dev_kfree_skb_any(skb);
1394                         goto next_desc;
1395                 }
1396
1397                 ixgbe_rx_checksum(adapter, rx_desc, skb, staterr);
1398                 if (adapter->netdev->features & NETIF_F_RXHASH)
1399                         ixgbe_rx_hash(rx_desc, skb);
1400
1401                 /* probably a little skewed due to removing CRC */
1402                 total_rx_bytes += skb->len;
1403                 total_rx_packets++;
1404
1405                 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1406 #ifdef IXGBE_FCOE
1407                 /* if ddp, not passing to ULD unless for FCP_RSP or error */
1408                 if (ixgbe_rx_is_fcoe(adapter, rx_desc)) {
1409                         ddp_bytes = ixgbe_fcoe_ddp(adapter, rx_desc, skb,
1410                                                    staterr);
1411                         if (!ddp_bytes) {
1412                                 dev_kfree_skb_any(skb);
1413                                 goto next_desc;
1414                         }
1415                 }
1416 #endif /* IXGBE_FCOE */
1417                 ixgbe_receive_skb(q_vector, skb, staterr, rx_ring, rx_desc);
1418
1419                 budget--;
1420 next_desc:
1421                 rx_desc->wb.upper.status_error = 0;
1422
1423                 if (!budget)
1424                         break;
1425
1426                 /* return some buffers to hardware, one at a time is too slow */
1427                 if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) {
1428                         ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
1429                         cleaned_count = 0;
1430                 }
1431
1432                 /* use prefetched values */
1433                 rx_desc = next_rxd;
1434                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1435         }
1436
1437         rx_ring->next_to_clean = i;
1438         cleaned_count = ixgbe_desc_unused(rx_ring);
1439
1440         if (cleaned_count)
1441                 ixgbe_alloc_rx_buffers(rx_ring, cleaned_count);
1442
1443 #ifdef IXGBE_FCOE
1444         /* include DDPed FCoE data */
1445         if (ddp_bytes > 0) {
1446                 unsigned int mss;
1447
1448                 mss = rx_ring->netdev->mtu - sizeof(struct fcoe_hdr) -
1449                         sizeof(struct fc_frame_header) -
1450                         sizeof(struct fcoe_crc_eof);
1451                 if (mss > 512)
1452                         mss &= ~511;
1453                 total_rx_bytes += ddp_bytes;
1454                 total_rx_packets += DIV_ROUND_UP(ddp_bytes, mss);
1455         }
1456 #endif /* IXGBE_FCOE */
1457
1458         u64_stats_update_begin(&rx_ring->syncp);
1459         rx_ring->stats.packets += total_rx_packets;
1460         rx_ring->stats.bytes += total_rx_bytes;
1461         u64_stats_update_end(&rx_ring->syncp);
1462         q_vector->rx.total_packets += total_rx_packets;
1463         q_vector->rx.total_bytes += total_rx_bytes;
1464
1465         return !!budget;
1466 }
1467
1468 /**
1469  * ixgbe_configure_msix - Configure MSI-X hardware
1470  * @adapter: board private structure
1471  *
1472  * ixgbe_configure_msix sets up the hardware to properly generate MSI-X
1473  * interrupts.
1474  **/
1475 static void ixgbe_configure_msix(struct ixgbe_adapter *adapter)
1476 {
1477         struct ixgbe_q_vector *q_vector;
1478         int q_vectors, v_idx;
1479         u32 mask;
1480
1481         q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
1482
1483         /* Populate MSIX to EITR Select */
1484         if (adapter->num_vfs > 32) {
1485                 u32 eitrsel = (1 << (adapter->num_vfs - 32)) - 1;
1486                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITRSEL, eitrsel);
1487         }
1488
1489         /*
1490          * Populate the IVAR table and set the ITR values to the
1491          * corresponding register.
1492          */
1493         for (v_idx = 0; v_idx < q_vectors; v_idx++) {
1494                 struct ixgbe_ring *ring;
1495                 q_vector = adapter->q_vector[v_idx];
1496
1497                 for (ring = q_vector->rx.ring; ring != NULL; ring = ring->next)
1498                         ixgbe_set_ivar(adapter, 0, ring->reg_idx, v_idx);
1499
1500                 for (ring = q_vector->tx.ring; ring != NULL; ring = ring->next)
1501                         ixgbe_set_ivar(adapter, 1, ring->reg_idx, v_idx);
1502
1503                 if (q_vector->tx.ring && !q_vector->rx.ring)
1504                         /* tx only */
1505                         q_vector->eitr = adapter->tx_eitr_param;
1506                 else if (q_vector->rx.ring)
1507                         /* rx or mixed */
1508                         q_vector->eitr = adapter->rx_eitr_param;
1509
1510                 ixgbe_write_eitr(q_vector);
1511         }
1512
1513         switch (adapter->hw.mac.type) {
1514         case ixgbe_mac_82598EB:
1515                 ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX,
1516                                v_idx);
1517                 break;
1518         case ixgbe_mac_82599EB:
1519         case ixgbe_mac_X540:
1520                 ixgbe_set_ivar(adapter, -1, 1, v_idx);
1521                 break;
1522
1523         default:
1524                 break;
1525         }
1526         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950);
1527
1528         /* set up to autoclear timer, and the vectors */
1529         mask = IXGBE_EIMS_ENABLE_MASK;
1530         if (adapter->num_vfs)
1531                 mask &= ~(IXGBE_EIMS_OTHER |
1532                           IXGBE_EIMS_MAILBOX |
1533                           IXGBE_EIMS_LSC);
1534         else
1535                 mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC);
1536         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask);
1537 }
1538
1539 enum latency_range {
1540         lowest_latency = 0,
1541         low_latency = 1,
1542         bulk_latency = 2,
1543         latency_invalid = 255
1544 };
1545
1546 /**
1547  * ixgbe_update_itr - update the dynamic ITR value based on statistics
1548  * @q_vector: structure containing interrupt and ring information
1549  * @ring_container: structure containing ring performance data
1550  *
1551  *      Stores a new ITR value based on packets and byte
1552  *      counts during the last interrupt.  The advantage of per interrupt
1553  *      computation is faster updates and more accurate ITR for the current
1554  *      traffic pattern.  Constants in this function were computed
1555  *      based on theoretical maximum wire speed and thresholds were set based
1556  *      on testing data as well as attempting to minimize response time
1557  *      while increasing bulk throughput.
1558  *      this functionality is controlled by the InterruptThrottleRate module
1559  *      parameter (see ixgbe_param.c)
1560  **/
1561 static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector,
1562                              struct ixgbe_ring_container *ring_container)
1563 {
1564         u64 bytes_perint;
1565         struct ixgbe_adapter *adapter = q_vector->adapter;
1566         int bytes = ring_container->total_bytes;
1567         int packets = ring_container->total_packets;
1568         u32 timepassed_us;
1569         u8 itr_setting = ring_container->itr;
1570
1571         if (packets == 0)
1572                 return;
1573
1574         /* simple throttlerate management
1575          *    0-20MB/s lowest (100000 ints/s)
1576          *   20-100MB/s low   (20000 ints/s)
1577          *  100-1249MB/s bulk (8000 ints/s)
1578          */
1579         /* what was last interrupt timeslice? */
1580         timepassed_us = 1000000/q_vector->eitr;
1581         bytes_perint = bytes / timepassed_us; /* bytes/usec */
1582
1583         switch (itr_setting) {
1584         case lowest_latency:
1585                 if (bytes_perint > adapter->eitr_low)
1586                         itr_setting = low_latency;
1587                 break;
1588         case low_latency:
1589                 if (bytes_perint > adapter->eitr_high)
1590                         itr_setting = bulk_latency;
1591                 else if (bytes_perint <= adapter->eitr_low)
1592                         itr_setting = lowest_latency;
1593                 break;
1594         case bulk_latency:
1595                 if (bytes_perint <= adapter->eitr_high)
1596                         itr_setting = low_latency;
1597                 break;
1598         }
1599
1600         /* clear work counters since we have the values we need */
1601         ring_container->total_bytes = 0;
1602         ring_container->total_packets = 0;
1603
1604         /* write updated itr to ring container */
1605         ring_container->itr = itr_setting;
1606 }
1607
1608 /**
1609  * ixgbe_write_eitr - write EITR register in hardware specific way
1610  * @q_vector: structure containing interrupt and ring information
1611  *
1612  * This function is made to be called by ethtool and by the driver
1613  * when it needs to update EITR registers at runtime.  Hardware
1614  * specific quirks/differences are taken care of here.
1615  */
1616 void ixgbe_write_eitr(struct ixgbe_q_vector *q_vector)
1617 {
1618         struct ixgbe_adapter *adapter = q_vector->adapter;
1619         struct ixgbe_hw *hw = &adapter->hw;
1620         int v_idx = q_vector->v_idx;
1621         u32 itr_reg = EITR_INTS_PER_SEC_TO_REG(q_vector->eitr);
1622
1623         switch (adapter->hw.mac.type) {
1624         case ixgbe_mac_82598EB:
1625                 /* must write high and low 16 bits to reset counter */
1626                 itr_reg |= (itr_reg << 16);
1627                 break;
1628         case ixgbe_mac_82599EB:
1629         case ixgbe_mac_X540:
1630                 /*
1631                  * 82599 and X540 can support a value of zero, so allow it for
1632                  * max interrupt rate, but there is an errata where it can
1633                  * not be zero with RSC
1634                  */
1635                 if (itr_reg == 8 &&
1636                     !(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED))
1637                         itr_reg = 0;
1638
1639                 /*
1640                  * set the WDIS bit to not clear the timer bits and cause an
1641                  * immediate assertion of the interrupt
1642                  */
1643                 itr_reg |= IXGBE_EITR_CNT_WDIS;
1644                 break;
1645         default:
1646                 break;
1647         }
1648         IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg);
1649 }
1650
1651 static void ixgbe_set_itr(struct ixgbe_q_vector *q_vector)
1652 {
1653         u32 new_itr = q_vector->eitr;
1654         u8 current_itr;
1655
1656         ixgbe_update_itr(q_vector, &q_vector->tx);
1657         ixgbe_update_itr(q_vector, &q_vector->rx);
1658
1659         current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
1660
1661         switch (current_itr) {
1662         /* counts and packets in update_itr are dependent on these numbers */
1663         case lowest_latency:
1664                 new_itr = 100000;
1665                 break;
1666         case low_latency:
1667                 new_itr = 20000; /* aka hwitr = ~200 */
1668                 break;
1669         case bulk_latency:
1670                 new_itr = 8000;
1671                 break;
1672         default:
1673                 break;
1674         }
1675
1676         if (new_itr != q_vector->eitr) {
1677                 /* do an exponential smoothing */
1678                 new_itr = ((q_vector->eitr * 9) + new_itr)/10;
1679
1680                 /* save the algorithm value here */
1681                 q_vector->eitr = new_itr;
1682
1683                 ixgbe_write_eitr(q_vector);
1684         }
1685 }
1686
1687 /**
1688  * ixgbe_check_overtemp_subtask - check for over tempurature
1689  * @adapter: pointer to adapter
1690  **/
1691 static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter)
1692 {
1693         struct ixgbe_hw *hw = &adapter->hw;
1694         u32 eicr = adapter->interrupt_event;
1695
1696         if (test_bit(__IXGBE_DOWN, &adapter->state))
1697                 return;
1698
1699         if (!(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
1700             !(adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_EVENT))
1701                 return;
1702
1703         adapter->flags2 &= ~IXGBE_FLAG2_TEMP_SENSOR_EVENT;
1704
1705         switch (hw->device_id) {
1706         case IXGBE_DEV_ID_82599_T3_LOM:
1707                 /*
1708                  * Since the warning interrupt is for both ports
1709                  * we don't have to check if:
1710                  *  - This interrupt wasn't for our port.
1711                  *  - We may have missed the interrupt so always have to
1712                  *    check if we  got a LSC
1713                  */
1714                 if (!(eicr & IXGBE_EICR_GPI_SDP0) &&
1715                     !(eicr & IXGBE_EICR_LSC))
1716                         return;
1717
1718                 if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) {
1719                         u32 autoneg;
1720                         bool link_up = false;
1721
1722                         hw->mac.ops.check_link(hw, &autoneg, &link_up, false);
1723
1724                         if (link_up)
1725                                 return;
1726                 }
1727
1728                 /* Check if this is not due to overtemp */
1729                 if (hw->phy.ops.check_overtemp(hw) != IXGBE_ERR_OVERTEMP)
1730                         return;
1731
1732                 break;
1733         default:
1734                 if (!(eicr & IXGBE_EICR_GPI_SDP0))
1735                         return;
1736                 break;
1737         }
1738         e_crit(drv,
1739                "Network adapter has been stopped because it has over heated. "
1740                "Restart the computer. If the problem persists, "
1741                "power off the system and replace the adapter\n");
1742
1743         adapter->interrupt_event = 0;
1744 }
1745
1746 static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr)
1747 {
1748         struct ixgbe_hw *hw = &adapter->hw;
1749
1750         if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) &&
1751             (eicr & IXGBE_EICR_GPI_SDP1)) {
1752                 e_crit(probe, "Fan has stopped, replace the adapter\n");
1753                 /* write to clear the interrupt */
1754                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1755         }
1756 }
1757
1758 static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr)
1759 {
1760         struct ixgbe_hw *hw = &adapter->hw;
1761
1762         if (eicr & IXGBE_EICR_GPI_SDP2) {
1763                 /* Clear the interrupt */
1764                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2);
1765                 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1766                         adapter->flags2 |= IXGBE_FLAG2_SFP_NEEDS_RESET;
1767                         ixgbe_service_event_schedule(adapter);
1768                 }
1769         }
1770
1771         if (eicr & IXGBE_EICR_GPI_SDP1) {
1772                 /* Clear the interrupt */
1773                 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1);
1774                 if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1775                         adapter->flags |= IXGBE_FLAG_NEED_LINK_CONFIG;
1776                         ixgbe_service_event_schedule(adapter);
1777                 }
1778         }
1779 }
1780
1781 static void ixgbe_check_lsc(struct ixgbe_adapter *adapter)
1782 {
1783         struct ixgbe_hw *hw = &adapter->hw;
1784
1785         adapter->lsc_int++;
1786         adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE;
1787         adapter->link_check_timeout = jiffies;
1788         if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1789                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC);
1790                 IXGBE_WRITE_FLUSH(hw);
1791                 ixgbe_service_event_schedule(adapter);
1792         }
1793 }
1794
1795 static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter,
1796                                            u64 qmask)
1797 {
1798         u32 mask;
1799         struct ixgbe_hw *hw = &adapter->hw;
1800
1801         switch (hw->mac.type) {
1802         case ixgbe_mac_82598EB:
1803                 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
1804                 IXGBE_WRITE_REG(hw, IXGBE_EIMS, mask);
1805                 break;
1806         case ixgbe_mac_82599EB:
1807         case ixgbe_mac_X540:
1808                 mask = (qmask & 0xFFFFFFFF);
1809                 if (mask)
1810                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(0), mask);
1811                 mask = (qmask >> 32);
1812                 if (mask)
1813                         IXGBE_WRITE_REG(hw, IXGBE_EIMS_EX(1), mask);
1814                 break;
1815         default:
1816                 break;
1817         }
1818         /* skip the flush */
1819 }
1820
1821 static inline void ixgbe_irq_disable_queues(struct ixgbe_adapter *adapter,
1822                                             u64 qmask)
1823 {
1824         u32 mask;
1825         struct ixgbe_hw *hw = &adapter->hw;
1826
1827         switch (hw->mac.type) {
1828         case ixgbe_mac_82598EB:
1829                 mask = (IXGBE_EIMS_RTX_QUEUE & qmask);
1830                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, mask);
1831                 break;
1832         case ixgbe_mac_82599EB:
1833         case ixgbe_mac_X540:
1834                 mask = (qmask & 0xFFFFFFFF);
1835                 if (mask)
1836                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(0), mask);
1837                 mask = (qmask >> 32);
1838                 if (mask)
1839                         IXGBE_WRITE_REG(hw, IXGBE_EIMC_EX(1), mask);
1840                 break;
1841         default:
1842                 break;
1843         }
1844         /* skip the flush */
1845 }
1846
1847 /**
1848  * ixgbe_irq_enable - Enable default interrupt generation settings
1849  * @adapter: board private structure
1850  **/
1851 static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues,
1852                                     bool flush)
1853 {
1854         u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE);
1855
1856         /* don't reenable LSC while waiting for link */
1857         if (adapter->flags & IXGBE_FLAG_NEED_LINK_UPDATE)
1858                 mask &= ~IXGBE_EIMS_LSC;
1859
1860         if (adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE)
1861                 mask |= IXGBE_EIMS_GPI_SDP0;
1862         if (adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE)
1863                 mask |= IXGBE_EIMS_GPI_SDP1;
1864         switch (adapter->hw.mac.type) {
1865         case ixgbe_mac_82599EB:
1866                 mask |= IXGBE_EIMS_GPI_SDP1;
1867                 mask |= IXGBE_EIMS_GPI_SDP2;
1868         case ixgbe_mac_X540:
1869                 mask |= IXGBE_EIMS_ECC;
1870                 mask |= IXGBE_EIMS_MAILBOX;
1871                 break;
1872         default:
1873                 break;
1874         }
1875         if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
1876             !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT))
1877                 mask |= IXGBE_EIMS_FLOW_DIR;
1878
1879         IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask);
1880         if (queues)
1881                 ixgbe_irq_enable_queues(adapter, ~0);
1882         if (flush)
1883                 IXGBE_WRITE_FLUSH(&adapter->hw);
1884 }
1885
1886 static irqreturn_t ixgbe_msix_other(int irq, void *data)
1887 {
1888         struct ixgbe_adapter *adapter = data;
1889         struct ixgbe_hw *hw = &adapter->hw;
1890         u32 eicr;
1891
1892         /*
1893          * Workaround for Silicon errata.  Use clear-by-write instead
1894          * of clear-by-read.  Reading with EICS will return the
1895          * interrupt causes without clearing, which later be done
1896          * with the write to EICR.
1897          */
1898         eicr = IXGBE_READ_REG(hw, IXGBE_EICS);
1899         IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr);
1900
1901         if (eicr & IXGBE_EICR_LSC)
1902                 ixgbe_check_lsc(adapter);
1903
1904         if (eicr & IXGBE_EICR_MAILBOX)
1905                 ixgbe_msg_task(adapter);
1906
1907         switch (hw->mac.type) {
1908         case ixgbe_mac_82599EB:
1909         case ixgbe_mac_X540:
1910                 if (eicr & IXGBE_EICR_ECC)
1911                         e_info(link, "Received unrecoverable ECC Err, please "
1912                                "reboot\n");
1913                 /* Handle Flow Director Full threshold interrupt */
1914                 if (eicr & IXGBE_EICR_FLOW_DIR) {
1915                         int reinit_count = 0;
1916                         int i;
1917                         for (i = 0; i < adapter->num_tx_queues; i++) {
1918                                 struct ixgbe_ring *ring = adapter->tx_ring[i];
1919                                 if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE,
1920                                                        &ring->state))
1921                                         reinit_count++;
1922                         }
1923                         if (reinit_count) {
1924                                 /* no more flow director interrupts until after init */
1925                                 IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_FLOW_DIR);
1926                                 adapter->flags2 |= IXGBE_FLAG2_FDIR_REQUIRES_REINIT;
1927                                 ixgbe_service_event_schedule(adapter);
1928                         }
1929                 }
1930                 ixgbe_check_sfp_event(adapter, eicr);
1931                 if ((adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
1932                     ((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC))) {
1933                         if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
1934                                 adapter->interrupt_event = eicr;
1935                                 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
1936                                 ixgbe_service_event_schedule(adapter);
1937                         }
1938                 }
1939                 break;
1940         default:
1941                 break;
1942         }
1943
1944         ixgbe_check_fan_failure(adapter, eicr);
1945
1946         /* re-enable the original interrupt state, no lsc, no queues */
1947         if (!test_bit(__IXGBE_DOWN, &adapter->state))
1948                 ixgbe_irq_enable(adapter, false, false);
1949
1950         return IRQ_HANDLED;
1951 }
1952
1953 static irqreturn_t ixgbe_msix_clean_rings(int irq, void *data)
1954 {
1955         struct ixgbe_q_vector *q_vector = data;
1956
1957         /* EIAM disabled interrupts (on this vector) for us */
1958
1959         if (q_vector->rx.ring || q_vector->tx.ring)
1960                 napi_schedule(&q_vector->napi);
1961
1962         return IRQ_HANDLED;
1963 }
1964
1965 static inline void map_vector_to_rxq(struct ixgbe_adapter *a, int v_idx,
1966                                      int r_idx)
1967 {
1968         struct ixgbe_q_vector *q_vector = a->q_vector[v_idx];
1969         struct ixgbe_ring *rx_ring = a->rx_ring[r_idx];
1970
1971         rx_ring->q_vector = q_vector;
1972         rx_ring->next = q_vector->rx.ring;
1973         q_vector->rx.ring = rx_ring;
1974         q_vector->rx.count++;
1975 }
1976
1977 static inline void map_vector_to_txq(struct ixgbe_adapter *a, int v_idx,
1978                                      int t_idx)
1979 {
1980         struct ixgbe_q_vector *q_vector = a->q_vector[v_idx];
1981         struct ixgbe_ring *tx_ring = a->tx_ring[t_idx];
1982
1983         tx_ring->q_vector = q_vector;
1984         tx_ring->next = q_vector->tx.ring;
1985         q_vector->tx.ring = tx_ring;
1986         q_vector->tx.count++;
1987         q_vector->tx.work_limit = a->tx_work_limit;
1988 }
1989
1990 /**
1991  * ixgbe_map_rings_to_vectors - Maps descriptor rings to vectors
1992  * @adapter: board private structure to initialize
1993  *
1994  * This function maps descriptor rings to the queue-specific vectors
1995  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
1996  * one vector per ring/queue, but on a constrained vector budget, we
1997  * group the rings as "efficiently" as possible.  You would add new
1998  * mapping configurations in here.
1999  **/
2000 static void ixgbe_map_rings_to_vectors(struct ixgbe_adapter *adapter)
2001 {
2002         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2003         int rxr_remaining = adapter->num_rx_queues, rxr_idx = 0;
2004         int txr_remaining = adapter->num_tx_queues, txr_idx = 0;
2005         int v_start = 0;
2006
2007         /* only one q_vector if MSI-X is disabled. */
2008         if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
2009                 q_vectors = 1;
2010
2011         /*
2012          * If we don't have enough vectors for a 1-to-1 mapping, we'll have to
2013          * group them so there are multiple queues per vector.
2014          *
2015          * Re-adjusting *qpv takes care of the remainder.
2016          */
2017         for (; v_start < q_vectors && rxr_remaining; v_start++) {
2018                 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_start);
2019                 for (; rqpv; rqpv--, rxr_idx++, rxr_remaining--)
2020                         map_vector_to_rxq(adapter, v_start, rxr_idx);
2021         }
2022
2023         /*
2024          * If there are not enough q_vectors for each ring to have it's own
2025          * vector then we must pair up Rx/Tx on a each vector
2026          */
2027         if ((v_start + txr_remaining) > q_vectors)
2028                 v_start = 0;
2029
2030         for (; v_start < q_vectors && txr_remaining; v_start++) {
2031                 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_start);
2032                 for (; tqpv; tqpv--, txr_idx++, txr_remaining--)
2033                         map_vector_to_txq(adapter, v_start, txr_idx);
2034         }
2035 }
2036
2037 /**
2038  * ixgbe_request_msix_irqs - Initialize MSI-X interrupts
2039  * @adapter: board private structure
2040  *
2041  * ixgbe_request_msix_irqs allocates MSI-X vectors and requests
2042  * interrupts from the kernel.
2043  **/
2044 static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter)
2045 {
2046         struct net_device *netdev = adapter->netdev;
2047         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2048         int vector, err;
2049         int ri = 0, ti = 0;
2050
2051         for (vector = 0; vector < q_vectors; vector++) {
2052                 struct ixgbe_q_vector *q_vector = adapter->q_vector[vector];
2053                 struct msix_entry *entry = &adapter->msix_entries[vector];
2054
2055                 if (q_vector->tx.ring && q_vector->rx.ring) {
2056                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2057                                  "%s-%s-%d", netdev->name, "TxRx", ri++);
2058                         ti++;
2059                 } else if (q_vector->rx.ring) {
2060                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2061                                  "%s-%s-%d", netdev->name, "rx", ri++);
2062                 } else if (q_vector->tx.ring) {
2063                         snprintf(q_vector->name, sizeof(q_vector->name) - 1,
2064                                  "%s-%s-%d", netdev->name, "tx", ti++);
2065                 } else {
2066                         /* skip this unused q_vector */
2067                         continue;
2068                 }
2069                 err = request_irq(entry->vector, &ixgbe_msix_clean_rings, 0,
2070                                   q_vector->name, q_vector);
2071                 if (err) {
2072                         e_err(probe, "request_irq failed for MSIX interrupt "
2073                               "Error: %d\n", err);
2074                         goto free_queue_irqs;
2075                 }
2076                 /* If Flow Director is enabled, set interrupt affinity */
2077                 if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) {
2078                         /* assign the mask for this irq */
2079                         irq_set_affinity_hint(entry->vector,
2080                                               q_vector->affinity_mask);
2081                 }
2082         }
2083
2084         err = request_irq(adapter->msix_entries[vector].vector,
2085                           ixgbe_msix_other, 0, netdev->name, adapter);
2086         if (err) {
2087                 e_err(probe, "request_irq for msix_lsc failed: %d\n", err);
2088                 goto free_queue_irqs;
2089         }
2090
2091         return 0;
2092
2093 free_queue_irqs:
2094         while (vector) {
2095                 vector--;
2096                 irq_set_affinity_hint(adapter->msix_entries[vector].vector,
2097                                       NULL);
2098                 free_irq(adapter->msix_entries[vector].vector,
2099                          adapter->q_vector[vector]);
2100         }
2101         adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED;
2102         pci_disable_msix(adapter->pdev);
2103         kfree(adapter->msix_entries);
2104         adapter->msix_entries = NULL;
2105         return err;
2106 }
2107
2108 /**
2109  * ixgbe_intr - legacy mode Interrupt Handler
2110  * @irq: interrupt number
2111  * @data: pointer to a network interface device structure
2112  **/
2113 static irqreturn_t ixgbe_intr(int irq, void *data)
2114 {
2115         struct ixgbe_adapter *adapter = data;
2116         struct ixgbe_hw *hw = &adapter->hw;
2117         struct ixgbe_q_vector *q_vector = adapter->q_vector[0];
2118         u32 eicr;
2119
2120         /*
2121          * Workaround for silicon errata on 82598.  Mask the interrupts
2122          * before the read of EICR.
2123          */
2124         IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_IRQ_CLEAR_MASK);
2125
2126         /* for NAPI, using EIAM to auto-mask tx/rx interrupt bits on read
2127          * therefore no explict interrupt disable is necessary */
2128         eicr = IXGBE_READ_REG(hw, IXGBE_EICR);
2129         if (!eicr) {
2130                 /*
2131                  * shared interrupt alert!
2132                  * make sure interrupts are enabled because the read will
2133                  * have disabled interrupts due to EIAM
2134                  * finish the workaround of silicon errata on 82598.  Unmask
2135                  * the interrupt that we masked before the EICR read.
2136                  */
2137                 if (!test_bit(__IXGBE_DOWN, &adapter->state))
2138                         ixgbe_irq_enable(adapter, true, true);
2139                 return IRQ_NONE;        /* Not our interrupt */
2140         }
2141
2142         if (eicr & IXGBE_EICR_LSC)
2143                 ixgbe_check_lsc(adapter);
2144
2145         switch (hw->mac.type) {
2146         case ixgbe_mac_82599EB:
2147                 ixgbe_check_sfp_event(adapter, eicr);
2148                 if ((adapter->flags2 & IXGBE_FLAG2_TEMP_SENSOR_CAPABLE) &&
2149                     ((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC))) {
2150                         if (!test_bit(__IXGBE_DOWN, &adapter->state)) {
2151                                 adapter->interrupt_event = eicr;
2152                                 adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT;
2153                                 ixgbe_service_event_schedule(adapter);
2154                         }
2155                 }
2156                 break;
2157         default:
2158                 break;
2159         }
2160
2161         ixgbe_check_fan_failure(adapter, eicr);
2162
2163         if (napi_schedule_prep(&(q_vector->napi))) {
2164                 /* would disable interrupts here but EIAM disabled it */
2165                 __napi_schedule(&(q_vector->napi));
2166         }
2167
2168         /*
2169          * re-enable link(maybe) and non-queue interrupts, no flush.
2170          * ixgbe_poll will re-enable the queue interrupts
2171          */
2172
2173         if (!test_bit(__IXGBE_DOWN, &adapter->state))
2174                 ixgbe_irq_enable(adapter, false, false);
2175
2176         return IRQ_HANDLED;
2177 }
2178
2179 static inline void ixgbe_reset_q_vectors(struct ixgbe_adapter *adapter)
2180 {
2181         int q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS;
2182         int i;
2183
2184         /* legacy and MSI only use one vector */
2185         if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED))
2186                 q_vectors = 1;
2187
2188         for (i = 0; i < adapter->num_rx_queues; i++) {
2189                 adapter->rx_ring[i]->q_vector = NULL;
2190                 adapter->rx_ring[i]->next = NULL;
2191         }
2192         for (i = 0; i < adapter->num_tx_queues; i++) {
2193                 adapter->tx_ring[i]->q_vector = NULL;
2194                 adapter->tx_ring[i]->next = NULL;
2195         }
2196
2197         for (i = 0; i < q_vectors; i++) {
2198                 struct ixgbe_q_vector *q_vector = adapter->q_vector[i];
2199                 memset(&q_vector->rx, 0, sizeof(struct ixgbe_ring_container));
2200                 memset(&q_vector->tx, 0, sizeof(struct ixgbe_ring_container));
2201         }
2202 }
2203
2204 /**
2205  * ixgbe_request_irq - initialize interrupts
2206  * @adapter: board private structure
2207  *
2208  * Attempts to configure interrupts using the best available
2209  * capabilities of the hardware and kernel.
2210  **/
2211 static int ixgbe_request_irq(struct ixgbe_adapter *adapter)
2212 {
2213         struct net_device *netdev = adapter->netdev;
2214         int err;
2215
2216         /* map all of the rings to the q_vectors */
2217         ixgbe_map_rings_to_vectors(adapter);
2218
2219         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED)
2220                 err = ixgbe_request_msix_irqs(adapter);
2221         else if (adapter->flags & IXGBE_FLAG_MSI_ENABLED)
2222                 err = request_irq(adapter->pdev->irq, ixgbe_intr, 0,
2223                                   netdev->name, adapter);
2224         else
2225                 err = request_irq(adapter->pdev->irq, ixgbe_intr, IRQF_SHARED,
2226                                   netdev->name, adapter);
2227
2228         if (err) {
2229                 e_err(probe, "request_irq failed, Error %d\n", err);
2230
2231                 /* place q_vectors and rings back into a known good state */
2232                 ixgbe_reset_q_vectors(adapter);
2233         }
2234
2235         return err;
2236 }
2237
2238 static void ixgbe_free_irq(struct ixgbe_adapter *adapter)
2239 {
2240         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
2241                 int i, q_vectors;
2242
2243                 q_vectors = adapter->num_msix_vectors;
2244                 i = q_vectors - 1;
2245                 free_irq(adapter->msix_entries[i].vector, adapter);
2246                 i--;
2247
2248                 for (; i >= 0; i--) {
2249                         /* free only the irqs that were actually requested */
2250                         if (!adapter->q_vector[i]->rx.ring &&
2251                             !adapter->q_vector[i]->tx.ring)
2252                                 continue;
2253
2254                         /* clear the affinity_mask in the IRQ descriptor */
2255                         irq_set_affinity_hint(adapter->msix_entries[i].vector,
2256                                               NULL);
2257
2258                         free_irq(adapter->msix_entries[i].vector,
2259                                  adapter->q_vector[i]);
2260                 }
2261         } else {
2262                 free_irq(adapter->pdev->irq, adapter);
2263         }
2264
2265         /* clear q_vector state information */
2266         ixgbe_reset_q_vectors(adapter);
2267 }
2268
2269 /**
2270  * ixgbe_irq_disable - Mask off interrupt generation on the NIC
2271  * @adapter: board private structure
2272  **/
2273 static inline void ixgbe_irq_disable(struct ixgbe_adapter *adapter)
2274 {
2275         switch (adapter->hw.mac.type) {
2276         case ixgbe_mac_82598EB:
2277                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, ~0);
2278                 break;
2279         case ixgbe_mac_82599EB:
2280         case ixgbe_mac_X540:
2281                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, 0xFFFF0000);
2282                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(0), ~0);
2283                 IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC_EX(1), ~0);
2284                 break;
2285         default:
2286                 break;
2287         }
2288         IXGBE_WRITE_FLUSH(&adapter->hw);
2289         if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) {
2290                 int i;
2291                 for (i = 0; i < adapter->num_msix_vectors; i++)
2292                         synchronize_irq(adapter->msix_entries[i].vector);
2293         } else {
2294                 synchronize_irq(adapter->pdev->irq);
2295         }
2296 }
2297
2298 /**
2299  * ixgbe_configure_msi_and_legacy - Initialize PIN (INTA...) and MSI interrupts
2300  *
2301  **/
2302 static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter)
2303 {
2304         struct ixgbe_hw *hw = &adapter->hw;
2305
2306         IXGBE_WRITE_REG(hw, IXGBE_EITR(0),
2307                         EITR_INTS_PER_SEC_TO_REG(adapter->rx_eitr_param));
2308
2309         ixgbe_set_ivar(adapter, 0, 0, 0);
2310         ixgbe_set_ivar(adapter, 1, 0, 0);
2311
2312         e_info(hw, "Legacy interrupt IVAR setup done\n");
2313 }
2314
2315 /**
2316  * ixgbe_configure_tx_ring - Configure 8259x Tx ring after Reset
2317  * @adapter: board private structure
2318  * @ring: structure containing ring specific data
2319  *
2320  * Configure the Tx descriptor ring after a reset.
2321  **/
2322 void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter,
2323                              struct ixgbe_ring *ring)
2324 {
2325         struct ixgbe_hw *hw = &adapter->hw;
2326         u64 tdba = ring->dma;
2327         int wait_loop = 10;
2328         u32 txdctl = IXGBE_TXDCTL_ENABLE;
2329         u8 reg_idx = ring->reg_idx;
2330
2331         /* disable queue to avoid issues while updating state */
2332         IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0);
2333         IXGBE_WRITE_FLUSH(hw);
2334
2335         IXGBE_WRITE_REG(hw, IXGBE_TDBAL(reg_idx),
2336                         (tdba & DMA_BIT_MASK(32)));
2337         IXGBE_WRITE_REG(hw, IXGBE_TDBAH(reg_idx), (tdba >> 32));
2338         IXGBE_WRITE_REG(hw, IXGBE_TDLEN(reg_idx),
2339                         ring->count * sizeof(union ixgbe_adv_tx_desc));
2340         IXGBE_WRITE_REG(hw, IXGBE_TDH(reg_idx), 0);
2341         IXGBE_WRITE_REG(hw, IXGBE_TDT(reg_idx), 0);
2342         ring->tail = hw->hw_addr + IXGBE_TDT(reg_idx);
2343
2344         /*
2345          * set WTHRESH to encourage burst writeback, it should not be set
2346          * higher than 1 when ITR is 0 as it could cause false TX hangs
2347          *
2348          * In order to avoid issues WTHRESH + PTHRESH should always be equal
2349          * to or less than the number of on chip descriptors, which is
2350          * currently 40.
2351          */
2352         if (!adapter->tx_itr_setting || !adapter->rx_itr_setting)
2353                 txdctl |= (1 << 16);    /* WTHRESH = 1 */
2354         else
2355                 txdctl |= (8 << 16);    /* WTHRESH = 8 */
2356
2357         /* PTHRESH=32 is needed to avoid a Tx hang with DFP enabled. */
2358         txdctl |= (1 << 8) |    /* HTHRESH = 1 */
2359                    32;          /* PTHRESH = 32 */
2360
2361         /* reinitialize flowdirector state */
2362         if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) &&
2363             adapter->atr_sample_rate) {
2364                 ring->atr_sample_rate = adapter->atr_sample_rate;
2365                 ring->atr_count = 0;
2366                 set_bit(__IXGBE_TX_FDIR_INIT_DONE, &ring->state);
2367         } else {
2368                 ring->atr_sample_rate = 0;
2369         }
2370
2371         clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state);
2372
2373         /* enable queue */
2374         IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl);
2375
2376         /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */
2377         if (hw->mac.type == ixgbe_mac_82598EB &&
2378             !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
2379                 return;
2380
2381         /* poll to verify queue is enabled */
2382         do {
2383                 usleep_range(1000, 2000);
2384                 txdctl = IXGBE_READ_REG(hw, IXGBE_TXDCTL(reg_idx));
2385         } while (--wait_loop && !(txdctl & IXGBE_TXDCTL_ENABLE));
2386         if (!wait_loop)
2387                 e_err(drv, "Could not enable Tx Queue %d\n", reg_idx);
2388 }
2389
2390 static void ixgbe_setup_mtqc(struct ixgbe_adapter *adapter)
2391 {
2392         struct ixgbe_hw *hw = &adapter->hw;
2393         u32 rttdcs;
2394         u32 reg;
2395         u8 tcs = netdev_get_num_tc(adapter->netdev);
2396
2397         if (hw->mac.type == ixgbe_mac_82598EB)
2398                 return;
2399
2400         /* disable the arbiter while setting MTQC */
2401         rttdcs = IXGBE_READ_REG(hw, IXGBE_RTTDCS);
2402         rttdcs |= IXGBE_RTTDCS_ARBDIS;
2403         IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2404
2405         /* set transmit pool layout */
2406         switch (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) {
2407         case (IXGBE_FLAG_SRIOV_ENABLED):
2408                 IXGBE_WRITE_REG(hw, IXGBE_MTQC,
2409                                 (IXGBE_MTQC_VT_ENA | IXGBE_MTQC_64VF));
2410                 break;
2411         default:
2412                 if (!tcs)
2413                         reg = IXGBE_MTQC_64Q_1PB;
2414                 else if (tcs <= 4)
2415                         reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_4TC_4TQ;
2416                 else
2417                         reg = IXGBE_MTQC_RT_ENA | IXGBE_MTQC_8TC_8TQ;
2418
2419                 IXGBE_WRITE_REG(hw, IXGBE_MTQC, reg);
2420
2421                 /* Enable Security TX Buffer IFG for multiple pb */
2422                 if (tcs) {
2423                         reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
2424                         reg |= IXGBE_SECTX_DCB;
2425                         IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
2426                 }
2427                 break;
2428         }
2429
2430         /* re-enable the arbiter */
2431         rttdcs &= ~IXGBE_RTTDCS_ARBDIS;
2432         IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, rttdcs);
2433 }
2434
2435 /**
2436  * ixgbe_configure_tx - Configure 8259x Transmit Unit after Reset
2437  * @adapter: board private structure
2438  *
2439  * Configure the Tx unit of the MAC after a reset.
2440  **/
2441 static void ixgbe_configure_tx(struct ixgbe_adapter *adapter)
2442 {
2443         struct ixgbe_hw *hw = &adapter->hw;
2444         u32 dmatxctl;
2445         u32 i;
2446
2447         ixgbe_setup_mtqc(adapter);
2448
2449         if (hw->mac.type != ixgbe_mac_82598EB) {
2450                 /* DMATXCTL.EN must be before Tx queues are enabled */
2451                 dmatxctl = IXGBE_READ_REG(hw, IXGBE_DMATXCTL);
2452                 dmatxctl |= IXGBE_DMATXCTL_TE;
2453                 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, dmatxctl);
2454         }
2455
2456         /* Setup the HW Tx Head and Tail descriptor pointers */
2457         for (i = 0; i < adapter->num_tx_queues; i++)
2458                 ixgbe_configure_tx_ring(adapter, adapter->tx_ring[i]);
2459 }
2460
2461 #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2
2462
2463 static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter,
2464                                    struct ixgbe_ring *rx_ring)
2465 {
2466         u32 srrctl;
2467         u8 reg_idx = rx_ring->reg_idx;
2468
2469         switch (adapter->hw.mac.type) {
2470         case ixgbe_mac_82598EB: {
2471                 struct ixgbe_ring_feature *feature = adapter->ring_feature;
2472                 const int mask = feature[RING_F_RSS].mask;
2473                 reg_idx = reg_idx & mask;
2474         }
2475                 break;
2476         case ixgbe_mac_82599EB:
2477         case ixgbe_mac_X540:
2478         default:
2479                 break;
2480         }
2481
2482         srrctl = IXGBE_READ_REG(&adapter->hw, IXGBE_SRRCTL(reg_idx));
2483
2484         srrctl &= ~IXGBE_SRRCTL_BSIZEHDR_MASK;
2485         srrctl &= ~IXGBE_SRRCTL_BSIZEPKT_MASK;
2486         if (adapter->num_vfs)
2487                 srrctl |= IXGBE_SRRCTL_DROP_EN;
2488
2489         srrctl |= (IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT) &
2490                   IXGBE_SRRCTL_BSIZEHDR_MASK;
2491
2492         if (ring_is_ps_enabled(rx_ring)) {
2493 #if (PAGE_SIZE / 2) > IXGBE_MAX_RXBUFFER
2494                 srrctl |= IXGBE_MAX_RXBUFFER >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2495 #else
2496                 srrctl |= (PAGE_SIZE / 2) >> IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2497 #endif
2498                 srrctl |= IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
2499         } else {
2500                 srrctl |= ALIGN(rx_ring->rx_buf_len, 1024) >>
2501                           IXGBE_SRRCTL_BSIZEPKT_SHIFT;
2502                 srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF;
2503         }
2504
2505         IXGBE_WRITE_REG(&adapter->hw, IXGBE_SRRCTL(reg_idx), srrctl);
2506 }
2507
2508 static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter)
2509 {
2510         struct ixgbe_hw *hw = &adapter->hw;
2511         static const u32 seed[10] = { 0xE291D73D, 0x1805EC6C, 0x2A94B30D,
2512                           0xA54F2BEC, 0xEA49AF7C, 0xE214AD3D, 0xB855AABE,
2513                           0x6A3E67EA, 0x14364D17, 0x3BED200D};
2514         u32 mrqc = 0, reta = 0;
2515         u32 rxcsum;
2516         int i, j;
2517         u8 tcs = netdev_get_num_tc(adapter->netdev);
2518         int maxq = adapter->ring_feature[RING_F_RSS].indices;
2519
2520         if (tcs)
2521                 maxq = min(maxq, adapter->num_tx_queues / tcs);
2522
2523         /* Fill out hash function seeds */
2524         for (i = 0; i < 10; i++)
2525                 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), seed[i]);
2526
2527         /* Fill out redirection table */
2528         for (i = 0, j = 0; i < 128; i++, j++) {
2529                 if (j == maxq)
2530                         j = 0;
2531                 /* reta = 4-byte sliding window of
2532                  * 0x00..(indices-1)(indices-1)00..etc. */
2533                 reta = (reta << 8) | (j * 0x11);
2534                 if ((i & 3) == 3)
2535                         IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta);
2536         }
2537
2538         /* Disable indicating checksum in descriptor, enables RSS hash */
2539         rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM);
2540         rxcsum |= IXGBE_RXCSUM_PCSD;
2541         IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum);
2542
2543         if (adapter->hw.mac.type == ixgbe_mac_82598EB &&
2544             (adapter->flags & IXGBE_FLAG_RSS_ENABLED)) {
2545                 mrqc = IXGBE_MRQC_RSSEN;
2546         } else {
2547                 int mask = adapter->flags & (IXGBE_FLAG_RSS_ENABLED
2548                                              | IXGBE_FLAG_SRIOV_ENABLED);
2549
2550                 switch (mask) {
2551                 case (IXGBE_FLAG_RSS_ENABLED):
2552                         if (!tcs)
2553                                 mrqc = IXGBE_MRQC_RSSEN;
2554                         else if (tcs <= 4)
2555                                 mrqc = IXGBE_MRQC_RTRSS4TCEN;
2556                         else
2557                                 mrqc = IXGBE_MRQC_RTRSS8TCEN;
2558                         break;
2559                 case (IXGBE_FLAG_SRIOV_ENABLED):
2560                         mrqc = IXGBE_MRQC_VMDQEN;
2561                         break;
2562                 default:
2563                         break;
2564                 }
2565         }
2566
2567         /* Perform hash on these packet types */
2568         mrqc |= IXGBE_MRQC_RSS_FIELD_IPV4
2569               | IXGBE_MRQC_RSS_FIELD_IPV4_TCP
2570               | IXGBE_MRQC_RSS_FIELD_IPV6
2571               | IXGBE_MRQC_RSS_FIELD_IPV6_TCP;
2572
2573         IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc);
2574 }
2575
2576 /**
2577  * ixgbe_configure_rscctl - enable RSC for the indicated ring
2578  * @adapter:    address of board private structure
2579  * @index:      index of ring to set
2580  **/
2581 static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter,
2582                                    struct ixgbe_ring *ring)
2583 {
2584         struct ixgbe_hw *hw = &adapter->hw;
2585         u32 rscctrl;
2586         int rx_buf_len;
2587         u8 reg_idx = ring->reg_idx;
2588
2589         if (!ring_is_rsc_enabled(ring))
2590                 return;
2591
2592         rx_buf_len = ring->rx_buf_len;
2593         rscctrl = IXGBE_READ_REG(hw, IXGBE_RSCCTL(reg_idx));
2594         rscctrl |= IXGBE_RSCCTL_RSCEN;
2595         /*
2596          * we must limit the number of descriptors so that the
2597          * total size of max desc * buf_len is not greater
2598          * than 65535
2599          */
2600         if (ring_is_ps_enabled(ring)) {
2601 #if (MAX_SKB_FRAGS > 16)
2602                 rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
2603 #elif (MAX_SKB_FRAGS > 8)
2604                 rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
2605 #elif (MAX_SKB_FRAGS > 4)
2606                 rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
2607 #else
2608                 rscctrl |= IXGBE_RSCCTL_MAXDESC_1;
2609 #endif
2610         } else {
2611                 if (rx_buf_len < IXGBE_RXBUFFER_4K)
2612                         rscctrl |= IXGBE_RSCCTL_MAXDESC_16;
2613                 else if (rx_buf_len < IXGBE_RXBUFFER_8K)
2614                         rscctrl |= IXGBE_RSCCTL_MAXDESC_8;
2615                 else
2616                         rscctrl |= IXGBE_RSCCTL_MAXDESC_4;
2617         }
2618         IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl);
2619 }
2620
2621 /**
2622  *  ixgbe_set_uta - Set unicast filter table address
2623  *  @adapter: board private structure
2624  *
2625  *  The unicast table address is a register array of 32-bit registers.
2626  *  The table is meant to be used in a way similar to how the MTA is used
2627  *  however due to certain limitations in the hardware it is necessary to
2628  *  set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
2629  *  enable bit to allow vlan tag stripping when promiscuous mode is enabled
2630  **/
2631 static void ixgbe_set_uta(struct ixgbe_adapter *adapter)
2632 {
2633         struct ixgbe_hw *hw = &adapter->hw;
2634         int i;
2635
2636         /* The UTA table only exists on 82599 hardware and newer */
2637         if (hw->mac.type < ixgbe_mac_82599EB)
2638                 return;
2639
2640         /* we only need to do this if VMDq is enabled */
2641         if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
2642                 return;
2643
2644         for (i = 0; i < 128; i++)
2645                 IXGBE_WRITE_REG(hw, IXGBE_UTA(i), ~0);
2646 }
2647
2648 #define IXGBE_MAX_RX_DESC_POLL 10
2649 static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter,
2650                                        struct ixgbe_ring *ring)
2651 {
2652         struct ixgbe_hw *hw = &adapter->hw;
2653         int wait_loop = IXGBE_MAX_RX_DESC_POLL;
2654         u32 rxdctl;
2655         u8 reg_idx = ring->reg_idx;
2656
2657         /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */
2658         if (hw->mac.type == ixgbe_mac_82598EB &&
2659             !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
2660                 return;
2661
2662         do {
2663                 usleep_range(1000, 2000);
2664                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2665         } while (--wait_loop && !(rxdctl & IXGBE_RXDCTL_ENABLE));
2666
2667         if (!wait_loop) {
2668                 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within "
2669                       "the polling period\n", reg_idx);
2670         }
2671 }
2672
2673 void ixgbe_disable_rx_queue(struct ixgbe_adapter *adapter,
2674                             struct ixgbe_ring *ring)
2675 {
2676         struct ixgbe_hw *hw = &adapter->hw;
2677         int wait_loop = IXGBE_MAX_RX_DESC_POLL;
2678         u32 rxdctl;
2679         u8 reg_idx = ring->reg_idx;
2680
2681         rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2682         rxdctl &= ~IXGBE_RXDCTL_ENABLE;
2683
2684         /* write value back with RXDCTL.ENABLE bit cleared */
2685         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
2686
2687         if (hw->mac.type == ixgbe_mac_82598EB &&
2688             !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP))
2689                 return;
2690
2691         /* the hardware may take up to 100us to really disable the rx queue */
2692         do {
2693                 udelay(10);
2694                 rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2695         } while (--wait_loop && (rxdctl & IXGBE_RXDCTL_ENABLE));
2696
2697         if (!wait_loop) {
2698                 e_err(drv, "RXDCTL.ENABLE on Rx queue %d not cleared within "
2699                       "the polling period\n", reg_idx);
2700         }
2701 }
2702
2703 void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter,
2704                              struct ixgbe_ring *ring)
2705 {
2706         struct ixgbe_hw *hw = &adapter->hw;
2707         u64 rdba = ring->dma;
2708         u32 rxdctl;
2709         u8 reg_idx = ring->reg_idx;
2710
2711         /* disable queue to avoid issues while updating state */
2712         rxdctl = IXGBE_READ_REG(hw, IXGBE_RXDCTL(reg_idx));
2713         ixgbe_disable_rx_queue(adapter, ring);
2714
2715         IXGBE_WRITE_REG(hw, IXGBE_RDBAL(reg_idx), (rdba & DMA_BIT_MASK(32)));
2716         IXGBE_WRITE_REG(hw, IXGBE_RDBAH(reg_idx), (rdba >> 32));
2717         IXGBE_WRITE_REG(hw, IXGBE_RDLEN(reg_idx),
2718                         ring->count * sizeof(union ixgbe_adv_rx_desc));
2719         IXGBE_WRITE_REG(hw, IXGBE_RDH(reg_idx), 0);
2720         IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0);
2721         ring->tail = hw->hw_addr + IXGBE_RDT(reg_idx);
2722
2723         ixgbe_configure_srrctl(adapter, ring);
2724         ixgbe_configure_rscctl(adapter, ring);
2725
2726         /* If operating in IOV mode set RLPML for X540 */
2727         if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) &&
2728             hw->mac.type == ixgbe_mac_X540) {
2729                 rxdctl &= ~IXGBE_RXDCTL_RLPMLMASK;
2730                 rxdctl |= ((ring->netdev->mtu + ETH_HLEN +
2731                             ETH_FCS_LEN + VLAN_HLEN) | IXGBE_RXDCTL_RLPML_EN);
2732         }
2733
2734         if (hw->mac.type == ixgbe_mac_82598EB) {
2735                 /*
2736                  * enable cache line friendly hardware writes:
2737                  * PTHRESH=32 descriptors (half the internal cache),
2738                  * this also removes ugly rx_no_buffer_count increment
2739                  * HTHRESH=4 descriptors (to minimize latency on fetch)
2740                  * WTHRESH=8 burst writeback up to two cache lines
2741                  */
2742                 rxdctl &= ~0x3FFFFF;
2743                 rxdctl |=  0x080420;
2744         }
2745
2746         /* enable receive descriptor ring */
2747         rxdctl |= IXGBE_RXDCTL_ENABLE;
2748         IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl);
2749
2750         ixgbe_rx_desc_queue_enable(adapter, ring);
2751         ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring));
2752 }
2753
2754 static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter)
2755 {
2756         struct ixgbe_hw *hw = &adapter->hw;
2757         int p;
2758
2759         /* PSRTYPE must be initialized in non 82598 adapters */
2760         u32 psrtype = IXGBE_PSRTYPE_TCPHDR |
2761                       IXGBE_PSRTYPE_UDPHDR |
2762                       IXGBE_PSRTYPE_IPV4HDR |
2763                       IXGBE_PSRTYPE_L2HDR |
2764                       IXGBE_PSRTYPE_IPV6HDR;
2765
2766         if (hw->mac.type == ixgbe_mac_82598EB)
2767                 return;
2768
2769         if (adapter->flags & IXGBE_FLAG_RSS_ENABLED)
2770                 psrtype |= (adapter->num_rx_queues_per_pool << 29);
2771
2772         for (p = 0; p < adapter->num_rx_pools; p++)
2773                 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(adapter->num_vfs + p),
2774                                 psrtype);
2775 }
2776
2777 static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter)
2778 {
2779         struct ixgbe_hw *hw = &adapter->hw;
2780         u32 gcr_ext;
2781         u32 vt_reg_bits;
2782         u32 reg_offset, vf_shift;
2783         u32 vmdctl;
2784
2785         if (!(adapter->flags & IXGBE_FLAG_SRIOV_ENABLED))
2786                 return;
2787
2788         vmdctl = IXGBE_READ_REG(hw, IXGBE_VT_CTL);
2789         vt_reg_bits = IXGBE_VMD_CTL_VMDQ_EN | IXGBE_VT_CTL_REPLEN;
2790         vt_reg_bits |= (adapter->num_vfs << IXGBE_VT_CTL_POOL_SHIFT);
2791         IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vmdctl | vt_reg_bits);
2792
2793         vf_shift = adapter->num_vfs % 32;
2794         reg_offset = (adapter->num_vfs > 32) ? 1 : 0;
2795
2796         /* Enable only the PF's pool for Tx/Rx */
2797         IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset), (1 << vf_shift));
2798         IXGBE_WRITE_REG(hw, IXGBE_VFRE(reg_offset ^ 1), 0);
2799         IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset), (1 << vf_shift));
2800         IXGBE_WRITE_REG(hw, IXGBE_VFTE(reg_offset ^ 1), 0);
2801         IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
2802
2803         /* Map PF MAC address in RAR Entry 0 to first pool following VFs */
2804         hw->mac.ops.set_vmdq(hw, 0, adapter->num_vfs);
2805
2806         /*
2807          * Set up VF register offsets for selected VT Mode,
2808          * i.e. 32 or 64 VFs for SR-IOV
2809          */
2810         gcr_ext = IXGBE_READ_REG(hw, IXGBE_GCR_EXT);
2811         gcr_ext |= IXGBE_GCR_EXT_MSIX_EN;
2812         gcr_ext |= IXGBE_GCR_EXT_VT_MODE_64;
2813         IXGBE_WRITE_REG(hw, IXGBE_GCR_EXT, gcr_ext);
2814
2815         /* enable Tx loopback for VF/PF communication */
2816         IXGBE_WRITE_REG(hw, IXGBE_PFDTXGSWC, IXGBE_PFDTXGSWC_VT_LBEN);
2817         /* Enable MAC Anti-Spoofing */
2818         hw->mac.ops.set_mac_anti_spoofing(hw,
2819                                           (adapter->antispoofing_enabled =
2820                                            (adapter->num_vfs != 0)),
2821                                           adapter->num_vfs);
2822 }
2823
2824 static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter)
2825 {
2826         struct ixgbe_hw *hw = &adapter->hw;
2827         struct net_device *netdev = adapter->netdev;
2828         int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
2829         int rx_buf_len;
2830         struct ixgbe_ring *rx_ring;
2831         int i;
2832         u32 mhadd, hlreg0;
2833
2834         /* Decide whether to use packet split mode or not */
2835         /* On by default */
2836         adapter->flags |= IXGBE_FLAG_RX_PS_ENABLED;
2837
2838         /* Do not use packet split if we're in SR-IOV Mode */
2839         if (adapter->num_vfs)
2840                 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED;
2841
2842         /* Disable packet split due to 82599 erratum #45 */
2843         if (hw->mac.type == ixgbe_mac_82599EB)
2844                 adapter->flags &= ~IXGBE_FLAG_RX_PS_ENABLED;
2845
2846 #ifdef IXGBE_FCOE
2847         /* adjust max frame to be able to do baby jumbo for FCoE */
2848         if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) &&
2849             (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE))
2850                 max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE;
2851
2852 #endif /* IXGBE_FCOE */
2853         mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD);
2854         if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) {
2855                 mhadd &= ~IXGBE_MHADD_MFS_MASK;
2856                 mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT;
2857
2858                 IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd);
2859         }
2860
2861         /* MHADD will allow an extra 4 bytes past for vlan tagged frames */
2862         max_frame += VLAN_HLEN;
2863
2864         /* Set the RX buffer length according to the mode */
2865         if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) {
2866                 rx_buf_len = IXGBE_RX_HDR_SIZE;
2867         } else {
2868                 if (!(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) &&
2869                     (netdev->mtu <= ETH_DATA_LEN))
2870                         rx_buf_len = MAXIMUM_ETHERNET_VLAN_SIZE;
2871                 /*
2872                  * Make best use of allocation by using all but 1K of a
2873                  * power of 2 allocation that will be used for skb->head.
2874                  */
2875                 else if (max_frame <= IXGBE_RXBUFFER_3K)
2876                         rx_buf_len = IXGBE_RXBUFFER_3K;
2877                 else if (max_frame <= IXGBE_RXBUFFER_7K)
2878                         rx_buf_len = IXGBE_RXBUFFER_7K;
2879                 else if (max_frame <= IXGBE_RXBUFFER_15K)
2880                         rx_buf_len = IXGBE_RXBUFFER_15K;
2881                 else
2882                         rx_buf_len = IXGBE_MAX_RXBUFFER;
2883         }
2884
2885         hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0);
2886         /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */
2887         hlreg0 |= IXGBE_HLREG0_JUMBOEN;
2888         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0);
2889
2890         /*
2891          * Setup the HW Rx Head and Tail Descriptor Pointers and
2892          * the Base and Length of the Rx Descriptor Ring
2893          */
2894         for (i = 0; i < adapter->num_rx_queues; i++) {
2895                 rx_ring = adapter->rx_ring[i];
2896                 rx_ring->rx_buf_len = rx_buf_len;
2897
2898                 if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED)
2899                         set_ring_ps_enabled(rx_ring);
2900                 else
2901                         clear_ring_ps_enabled(rx_ring);
2902
2903                 if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)
2904                         set_ring_rsc_enabled(rx_ring);
2905                 else
2906                         clear_ring_rsc_enabled(rx_ring);
2907
2908 #ifdef IXGBE_FCOE
2909                 if (netdev->features & NETIF_F_FCOE_MTU) {
2910                         struct ixgbe_ring_feature *f;
2911                         f = &adapter->ring_feature[RING_F_FCOE];
2912                         if ((i >= f->mask) && (i < f->mask + f->indices)) {
2913                                 clear_ring_ps_enabled(rx_ring);
2914                                 if (rx_buf_len < IXGBE_FCOE_JUMBO_FRAME_SIZE)
2915                                         rx_ring->rx_buf_len =
2916                                                 IXGBE_FCOE_JUMBO_FRAME_SIZE;
2917                         } else if (!ring_is_rsc_enabled(rx_ring) &&
2918                                    !ring_is_ps_enabled(rx_ring)) {
2919                                 rx_ring->rx_buf_len =
2920                                                 IXGBE_FCOE_JUMBO_FRAME_SIZE;
2921                         }
2922                 }
2923 #endif /* IXGBE_FCOE */
2924