Merge branch 'siocghwtstamp' of git://git.kernel.org/pub/scm/linux/kernel/git/bwh...
[linux-drm-fsl-dcu.git] / drivers / net / ethernet / intel / e1000e / netdev.c
1 /*******************************************************************************
2
3   Intel PRO/1000 Linux driver
4   Copyright(c) 1999 - 2013 Intel Corporation.
5
6   This program is free software; you can redistribute it and/or modify it
7   under the terms and conditions of the GNU General Public License,
8   version 2, as published by the Free Software Foundation.
9
10   This program is distributed in the hope it will be useful, but WITHOUT
11   ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12   FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
13   more details.
14
15   You should have received a copy of the GNU General Public License along with
16   this program; if not, write to the Free Software Foundation, Inc.,
17   51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19   The full GNU General Public License is included in this distribution in
20   the file called "COPYING".
21
22   Contact Information:
23   Linux NICS <linux.nics@intel.com>
24   e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
25   Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
26
27 *******************************************************************************/
28
29 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
30
31 #include <linux/module.h>
32 #include <linux/types.h>
33 #include <linux/init.h>
34 #include <linux/pci.h>
35 #include <linux/vmalloc.h>
36 #include <linux/pagemap.h>
37 #include <linux/delay.h>
38 #include <linux/netdevice.h>
39 #include <linux/interrupt.h>
40 #include <linux/tcp.h>
41 #include <linux/ipv6.h>
42 #include <linux/slab.h>
43 #include <net/checksum.h>
44 #include <net/ip6_checksum.h>
45 #include <linux/ethtool.h>
46 #include <linux/if_vlan.h>
47 #include <linux/cpu.h>
48 #include <linux/smp.h>
49 #include <linux/pm_qos.h>
50 #include <linux/pm_runtime.h>
51 #include <linux/aer.h>
52 #include <linux/prefetch.h>
53
54 #include "e1000.h"
55
56 #define DRV_EXTRAVERSION "-k"
57
58 #define DRV_VERSION "2.3.2" DRV_EXTRAVERSION
59 char e1000e_driver_name[] = "e1000e";
60 const char e1000e_driver_version[] = DRV_VERSION;
61
62 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
63 static int debug = -1;
64 module_param(debug, int, 0);
65 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
66
67 static const struct e1000_info *e1000_info_tbl[] = {
68         [board_82571]           = &e1000_82571_info,
69         [board_82572]           = &e1000_82572_info,
70         [board_82573]           = &e1000_82573_info,
71         [board_82574]           = &e1000_82574_info,
72         [board_82583]           = &e1000_82583_info,
73         [board_80003es2lan]     = &e1000_es2_info,
74         [board_ich8lan]         = &e1000_ich8_info,
75         [board_ich9lan]         = &e1000_ich9_info,
76         [board_ich10lan]        = &e1000_ich10_info,
77         [board_pchlan]          = &e1000_pch_info,
78         [board_pch2lan]         = &e1000_pch2_info,
79         [board_pch_lpt]         = &e1000_pch_lpt_info,
80 };
81
82 struct e1000_reg_info {
83         u32 ofs;
84         char *name;
85 };
86
87 static const struct e1000_reg_info e1000_reg_info_tbl[] = {
88         /* General Registers */
89         {E1000_CTRL, "CTRL"},
90         {E1000_STATUS, "STATUS"},
91         {E1000_CTRL_EXT, "CTRL_EXT"},
92
93         /* Interrupt Registers */
94         {E1000_ICR, "ICR"},
95
96         /* Rx Registers */
97         {E1000_RCTL, "RCTL"},
98         {E1000_RDLEN(0), "RDLEN"},
99         {E1000_RDH(0), "RDH"},
100         {E1000_RDT(0), "RDT"},
101         {E1000_RDTR, "RDTR"},
102         {E1000_RXDCTL(0), "RXDCTL"},
103         {E1000_ERT, "ERT"},
104         {E1000_RDBAL(0), "RDBAL"},
105         {E1000_RDBAH(0), "RDBAH"},
106         {E1000_RDFH, "RDFH"},
107         {E1000_RDFT, "RDFT"},
108         {E1000_RDFHS, "RDFHS"},
109         {E1000_RDFTS, "RDFTS"},
110         {E1000_RDFPC, "RDFPC"},
111
112         /* Tx Registers */
113         {E1000_TCTL, "TCTL"},
114         {E1000_TDBAL(0), "TDBAL"},
115         {E1000_TDBAH(0), "TDBAH"},
116         {E1000_TDLEN(0), "TDLEN"},
117         {E1000_TDH(0), "TDH"},
118         {E1000_TDT(0), "TDT"},
119         {E1000_TIDV, "TIDV"},
120         {E1000_TXDCTL(0), "TXDCTL"},
121         {E1000_TADV, "TADV"},
122         {E1000_TARC(0), "TARC"},
123         {E1000_TDFH, "TDFH"},
124         {E1000_TDFT, "TDFT"},
125         {E1000_TDFHS, "TDFHS"},
126         {E1000_TDFTS, "TDFTS"},
127         {E1000_TDFPC, "TDFPC"},
128
129         /* List Terminator */
130         {0, NULL}
131 };
132
133 /**
134  * e1000_regdump - register printout routine
135  * @hw: pointer to the HW structure
136  * @reginfo: pointer to the register info table
137  **/
138 static void e1000_regdump(struct e1000_hw *hw, struct e1000_reg_info *reginfo)
139 {
140         int n = 0;
141         char rname[16];
142         u32 regs[8];
143
144         switch (reginfo->ofs) {
145         case E1000_RXDCTL(0):
146                 for (n = 0; n < 2; n++)
147                         regs[n] = __er32(hw, E1000_RXDCTL(n));
148                 break;
149         case E1000_TXDCTL(0):
150                 for (n = 0; n < 2; n++)
151                         regs[n] = __er32(hw, E1000_TXDCTL(n));
152                 break;
153         case E1000_TARC(0):
154                 for (n = 0; n < 2; n++)
155                         regs[n] = __er32(hw, E1000_TARC(n));
156                 break;
157         default:
158                 pr_info("%-15s %08x\n",
159                         reginfo->name, __er32(hw, reginfo->ofs));
160                 return;
161         }
162
163         snprintf(rname, 16, "%s%s", reginfo->name, "[0-1]");
164         pr_info("%-15s %08x %08x\n", rname, regs[0], regs[1]);
165 }
166
167 static void e1000e_dump_ps_pages(struct e1000_adapter *adapter,
168                                  struct e1000_buffer *bi)
169 {
170         int i;
171         struct e1000_ps_page *ps_page;
172
173         for (i = 0; i < adapter->rx_ps_pages; i++) {
174                 ps_page = &bi->ps_pages[i];
175
176                 if (ps_page->page) {
177                         pr_info("packet dump for ps_page %d:\n", i);
178                         print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
179                                        16, 1, page_address(ps_page->page),
180                                        PAGE_SIZE, true);
181                 }
182         }
183 }
184
185 /**
186  * e1000e_dump - Print registers, Tx-ring and Rx-ring
187  * @adapter: board private structure
188  **/
189 static void e1000e_dump(struct e1000_adapter *adapter)
190 {
191         struct net_device *netdev = adapter->netdev;
192         struct e1000_hw *hw = &adapter->hw;
193         struct e1000_reg_info *reginfo;
194         struct e1000_ring *tx_ring = adapter->tx_ring;
195         struct e1000_tx_desc *tx_desc;
196         struct my_u0 {
197                 __le64 a;
198                 __le64 b;
199         } *u0;
200         struct e1000_buffer *buffer_info;
201         struct e1000_ring *rx_ring = adapter->rx_ring;
202         union e1000_rx_desc_packet_split *rx_desc_ps;
203         union e1000_rx_desc_extended *rx_desc;
204         struct my_u1 {
205                 __le64 a;
206                 __le64 b;
207                 __le64 c;
208                 __le64 d;
209         } *u1;
210         u32 staterr;
211         int i = 0;
212
213         if (!netif_msg_hw(adapter))
214                 return;
215
216         /* Print netdevice Info */
217         if (netdev) {
218                 dev_info(&adapter->pdev->dev, "Net device Info\n");
219                 pr_info("Device Name     state            trans_start      last_rx\n");
220                 pr_info("%-15s %016lX %016lX %016lX\n", netdev->name,
221                         netdev->state, netdev->trans_start, netdev->last_rx);
222         }
223
224         /* Print Registers */
225         dev_info(&adapter->pdev->dev, "Register Dump\n");
226         pr_info(" Register Name   Value\n");
227         for (reginfo = (struct e1000_reg_info *)e1000_reg_info_tbl;
228              reginfo->name; reginfo++) {
229                 e1000_regdump(hw, reginfo);
230         }
231
232         /* Print Tx Ring Summary */
233         if (!netdev || !netif_running(netdev))
234                 return;
235
236         dev_info(&adapter->pdev->dev, "Tx Ring Summary\n");
237         pr_info("Queue [NTU] [NTC] [bi(ntc)->dma  ] leng ntw timestamp\n");
238         buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
239         pr_info(" %5d %5X %5X %016llX %04X %3X %016llX\n",
240                 0, tx_ring->next_to_use, tx_ring->next_to_clean,
241                 (unsigned long long)buffer_info->dma,
242                 buffer_info->length,
243                 buffer_info->next_to_watch,
244                 (unsigned long long)buffer_info->time_stamp);
245
246         /* Print Tx Ring */
247         if (!netif_msg_tx_done(adapter))
248                 goto rx_ring_summary;
249
250         dev_info(&adapter->pdev->dev, "Tx Ring Dump\n");
251
252         /* Transmit Descriptor Formats - DEXT[29] is 0 (Legacy) or 1 (Extended)
253          *
254          * Legacy Transmit Descriptor
255          *   +--------------------------------------------------------------+
256          * 0 |         Buffer Address [63:0] (Reserved on Write Back)       |
257          *   +--------------------------------------------------------------+
258          * 8 | Special  |    CSS     | Status |  CMD    |  CSO   |  Length  |
259          *   +--------------------------------------------------------------+
260          *   63       48 47        36 35    32 31     24 23    16 15        0
261          *
262          * Extended Context Descriptor (DTYP=0x0) for TSO or checksum offload
263          *   63      48 47    40 39       32 31             16 15    8 7      0
264          *   +----------------------------------------------------------------+
265          * 0 |  TUCSE  | TUCS0  |   TUCSS   |     IPCSE       | IPCS0 | IPCSS |
266          *   +----------------------------------------------------------------+
267          * 8 |   MSS   | HDRLEN | RSV | STA | TUCMD | DTYP |      PAYLEN      |
268          *   +----------------------------------------------------------------+
269          *   63      48 47    40 39 36 35 32 31   24 23  20 19                0
270          *
271          * Extended Data Descriptor (DTYP=0x1)
272          *   +----------------------------------------------------------------+
273          * 0 |                     Buffer Address [63:0]                      |
274          *   +----------------------------------------------------------------+
275          * 8 | VLAN tag |  POPTS  | Rsvd | Status | Command | DTYP |  DTALEN  |
276          *   +----------------------------------------------------------------+
277          *   63       48 47     40 39  36 35    32 31     24 23  20 19        0
278          */
279         pr_info("Tl[desc]     [address 63:0  ] [SpeCssSCmCsLen] [bi->dma       ] leng  ntw timestamp        bi->skb <-- Legacy format\n");
280         pr_info("Tc[desc]     [Ce CoCsIpceCoS] [MssHlRSCm0Plen] [bi->dma       ] leng  ntw timestamp        bi->skb <-- Ext Context format\n");
281         pr_info("Td[desc]     [address 63:0  ] [VlaPoRSCm1Dlen] [bi->dma       ] leng  ntw timestamp        bi->skb <-- Ext Data format\n");
282         for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
283                 const char *next_desc;
284                 tx_desc = E1000_TX_DESC(*tx_ring, i);
285                 buffer_info = &tx_ring->buffer_info[i];
286                 u0 = (struct my_u0 *)tx_desc;
287                 if (i == tx_ring->next_to_use && i == tx_ring->next_to_clean)
288                         next_desc = " NTC/U";
289                 else if (i == tx_ring->next_to_use)
290                         next_desc = " NTU";
291                 else if (i == tx_ring->next_to_clean)
292                         next_desc = " NTC";
293                 else
294                         next_desc = "";
295                 pr_info("T%c[0x%03X]    %016llX %016llX %016llX %04X  %3X %016llX %p%s\n",
296                         (!(le64_to_cpu(u0->b) & (1 << 29)) ? 'l' :
297                          ((le64_to_cpu(u0->b) & (1 << 20)) ? 'd' : 'c')),
298                         i,
299                         (unsigned long long)le64_to_cpu(u0->a),
300                         (unsigned long long)le64_to_cpu(u0->b),
301                         (unsigned long long)buffer_info->dma,
302                         buffer_info->length, buffer_info->next_to_watch,
303                         (unsigned long long)buffer_info->time_stamp,
304                         buffer_info->skb, next_desc);
305
306                 if (netif_msg_pktdata(adapter) && buffer_info->skb)
307                         print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS,
308                                        16, 1, buffer_info->skb->data,
309                                        buffer_info->skb->len, true);
310         }
311
312         /* Print Rx Ring Summary */
313 rx_ring_summary:
314         dev_info(&adapter->pdev->dev, "Rx Ring Summary\n");
315         pr_info("Queue [NTU] [NTC]\n");
316         pr_info(" %5d %5X %5X\n",
317                 0, rx_ring->next_to_use, rx_ring->next_to_clean);
318
319         /* Print Rx Ring */
320         if (!netif_msg_rx_status(adapter))
321                 return;
322
323         dev_info(&adapter->pdev->dev, "Rx Ring Dump\n");
324         switch (adapter->rx_ps_pages) {
325         case 1:
326         case 2:
327         case 3:
328                 /* [Extended] Packet Split Receive Descriptor Format
329                  *
330                  *    +-----------------------------------------------------+
331                  *  0 |                Buffer Address 0 [63:0]              |
332                  *    +-----------------------------------------------------+
333                  *  8 |                Buffer Address 1 [63:0]              |
334                  *    +-----------------------------------------------------+
335                  * 16 |                Buffer Address 2 [63:0]              |
336                  *    +-----------------------------------------------------+
337                  * 24 |                Buffer Address 3 [63:0]              |
338                  *    +-----------------------------------------------------+
339                  */
340                 pr_info("R  [desc]      [buffer 0 63:0 ] [buffer 1 63:0 ] [buffer 2 63:0 ] [buffer 3 63:0 ] [bi->dma       ] [bi->skb] <-- Ext Pkt Split format\n");
341                 /* [Extended] Receive Descriptor (Write-Back) Format
342                  *
343                  *   63       48 47    32 31     13 12    8 7    4 3        0
344                  *   +------------------------------------------------------+
345                  * 0 | Packet   | IP     |  Rsvd   | MRQ   | Rsvd | MRQ RSS |
346                  *   | Checksum | Ident  |         | Queue |      |  Type   |
347                  *   +------------------------------------------------------+
348                  * 8 | VLAN Tag | Length | Extended Error | Extended Status |
349                  *   +------------------------------------------------------+
350                  *   63       48 47    32 31            20 19               0
351                  */
352                 pr_info("RWB[desc]      [ck ipid mrqhsh] [vl   l0 ee  es] [ l3  l2  l1 hs] [reserved      ] ---------------- [bi->skb] <-- Ext Rx Write-Back format\n");
353                 for (i = 0; i < rx_ring->count; i++) {
354                         const char *next_desc;
355                         buffer_info = &rx_ring->buffer_info[i];
356                         rx_desc_ps = E1000_RX_DESC_PS(*rx_ring, i);
357                         u1 = (struct my_u1 *)rx_desc_ps;
358                         staterr =
359                             le32_to_cpu(rx_desc_ps->wb.middle.status_error);
360
361                         if (i == rx_ring->next_to_use)
362                                 next_desc = " NTU";
363                         else if (i == rx_ring->next_to_clean)
364                                 next_desc = " NTC";
365                         else
366                                 next_desc = "";
367
368                         if (staterr & E1000_RXD_STAT_DD) {
369                                 /* Descriptor Done */
370                                 pr_info("%s[0x%03X]     %016llX %016llX %016llX %016llX ---------------- %p%s\n",
371                                         "RWB", i,
372                                         (unsigned long long)le64_to_cpu(u1->a),
373                                         (unsigned long long)le64_to_cpu(u1->b),
374                                         (unsigned long long)le64_to_cpu(u1->c),
375                                         (unsigned long long)le64_to_cpu(u1->d),
376                                         buffer_info->skb, next_desc);
377                         } else {
378                                 pr_info("%s[0x%03X]     %016llX %016llX %016llX %016llX %016llX %p%s\n",
379                                         "R  ", i,
380                                         (unsigned long long)le64_to_cpu(u1->a),
381                                         (unsigned long long)le64_to_cpu(u1->b),
382                                         (unsigned long long)le64_to_cpu(u1->c),
383                                         (unsigned long long)le64_to_cpu(u1->d),
384                                         (unsigned long long)buffer_info->dma,
385                                         buffer_info->skb, next_desc);
386
387                                 if (netif_msg_pktdata(adapter))
388                                         e1000e_dump_ps_pages(adapter,
389                                                              buffer_info);
390                         }
391                 }
392                 break;
393         default:
394         case 0:
395                 /* Extended Receive Descriptor (Read) Format
396                  *
397                  *   +-----------------------------------------------------+
398                  * 0 |                Buffer Address [63:0]                |
399                  *   +-----------------------------------------------------+
400                  * 8 |                      Reserved                       |
401                  *   +-----------------------------------------------------+
402                  */
403                 pr_info("R  [desc]      [buf addr 63:0 ] [reserved 63:0 ] [bi->dma       ] [bi->skb] <-- Ext (Read) format\n");
404                 /* Extended Receive Descriptor (Write-Back) Format
405                  *
406                  *   63       48 47    32 31    24 23            4 3        0
407                  *   +------------------------------------------------------+
408                  *   |     RSS Hash      |        |               |         |
409                  * 0 +-------------------+  Rsvd  |   Reserved    | MRQ RSS |
410                  *   | Packet   | IP     |        |               |  Type   |
411                  *   | Checksum | Ident  |        |               |         |
412                  *   +------------------------------------------------------+
413                  * 8 | VLAN Tag | Length | Extended Error | Extended Status |
414                  *   +------------------------------------------------------+
415                  *   63       48 47    32 31            20 19               0
416                  */
417                 pr_info("RWB[desc]      [cs ipid    mrq] [vt   ln xe  xs] [bi->skb] <-- Ext (Write-Back) format\n");
418
419                 for (i = 0; i < rx_ring->count; i++) {
420                         const char *next_desc;
421
422                         buffer_info = &rx_ring->buffer_info[i];
423                         rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
424                         u1 = (struct my_u1 *)rx_desc;
425                         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
426
427                         if (i == rx_ring->next_to_use)
428                                 next_desc = " NTU";
429                         else if (i == rx_ring->next_to_clean)
430                                 next_desc = " NTC";
431                         else
432                                 next_desc = "";
433
434                         if (staterr & E1000_RXD_STAT_DD) {
435                                 /* Descriptor Done */
436                                 pr_info("%s[0x%03X]     %016llX %016llX ---------------- %p%s\n",
437                                         "RWB", i,
438                                         (unsigned long long)le64_to_cpu(u1->a),
439                                         (unsigned long long)le64_to_cpu(u1->b),
440                                         buffer_info->skb, next_desc);
441                         } else {
442                                 pr_info("%s[0x%03X]     %016llX %016llX %016llX %p%s\n",
443                                         "R  ", i,
444                                         (unsigned long long)le64_to_cpu(u1->a),
445                                         (unsigned long long)le64_to_cpu(u1->b),
446                                         (unsigned long long)buffer_info->dma,
447                                         buffer_info->skb, next_desc);
448
449                                 if (netif_msg_pktdata(adapter) &&
450                                     buffer_info->skb)
451                                         print_hex_dump(KERN_INFO, "",
452                                                        DUMP_PREFIX_ADDRESS, 16,
453                                                        1,
454                                                        buffer_info->skb->data,
455                                                        adapter->rx_buffer_len,
456                                                        true);
457                         }
458                 }
459         }
460 }
461
462 /**
463  * e1000_desc_unused - calculate if we have unused descriptors
464  **/
465 static int e1000_desc_unused(struct e1000_ring *ring)
466 {
467         if (ring->next_to_clean > ring->next_to_use)
468                 return ring->next_to_clean - ring->next_to_use - 1;
469
470         return ring->count + ring->next_to_clean - ring->next_to_use - 1;
471 }
472
473 /**
474  * e1000e_systim_to_hwtstamp - convert system time value to hw time stamp
475  * @adapter: board private structure
476  * @hwtstamps: time stamp structure to update
477  * @systim: unsigned 64bit system time value.
478  *
479  * Convert the system time value stored in the RX/TXSTMP registers into a
480  * hwtstamp which can be used by the upper level time stamping functions.
481  *
482  * The 'systim_lock' spinlock is used to protect the consistency of the
483  * system time value. This is needed because reading the 64 bit time
484  * value involves reading two 32 bit registers. The first read latches the
485  * value.
486  **/
487 static void e1000e_systim_to_hwtstamp(struct e1000_adapter *adapter,
488                                       struct skb_shared_hwtstamps *hwtstamps,
489                                       u64 systim)
490 {
491         u64 ns;
492         unsigned long flags;
493
494         spin_lock_irqsave(&adapter->systim_lock, flags);
495         ns = timecounter_cyc2time(&adapter->tc, systim);
496         spin_unlock_irqrestore(&adapter->systim_lock, flags);
497
498         memset(hwtstamps, 0, sizeof(*hwtstamps));
499         hwtstamps->hwtstamp = ns_to_ktime(ns);
500 }
501
502 /**
503  * e1000e_rx_hwtstamp - utility function which checks for Rx time stamp
504  * @adapter: board private structure
505  * @status: descriptor extended error and status field
506  * @skb: particular skb to include time stamp
507  *
508  * If the time stamp is valid, convert it into the timecounter ns value
509  * and store that result into the shhwtstamps structure which is passed
510  * up the network stack.
511  **/
512 static void e1000e_rx_hwtstamp(struct e1000_adapter *adapter, u32 status,
513                                struct sk_buff *skb)
514 {
515         struct e1000_hw *hw = &adapter->hw;
516         u64 rxstmp;
517
518         if (!(adapter->flags & FLAG_HAS_HW_TIMESTAMP) ||
519             !(status & E1000_RXDEXT_STATERR_TST) ||
520             !(er32(TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
521                 return;
522
523         /* The Rx time stamp registers contain the time stamp.  No other
524          * received packet will be time stamped until the Rx time stamp
525          * registers are read.  Because only one packet can be time stamped
526          * at a time, the register values must belong to this packet and
527          * therefore none of the other additional attributes need to be
528          * compared.
529          */
530         rxstmp = (u64)er32(RXSTMPL);
531         rxstmp |= (u64)er32(RXSTMPH) << 32;
532         e1000e_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), rxstmp);
533
534         adapter->flags2 &= ~FLAG2_CHECK_RX_HWTSTAMP;
535 }
536
537 /**
538  * e1000_receive_skb - helper function to handle Rx indications
539  * @adapter: board private structure
540  * @staterr: descriptor extended error and status field as written by hardware
541  * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
542  * @skb: pointer to sk_buff to be indicated to stack
543  **/
544 static void e1000_receive_skb(struct e1000_adapter *adapter,
545                               struct net_device *netdev, struct sk_buff *skb,
546                               u32 staterr, __le16 vlan)
547 {
548         u16 tag = le16_to_cpu(vlan);
549
550         e1000e_rx_hwtstamp(adapter, staterr, skb);
551
552         skb->protocol = eth_type_trans(skb, netdev);
553
554         if (staterr & E1000_RXD_STAT_VP)
555                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), tag);
556
557         napi_gro_receive(&adapter->napi, skb);
558 }
559
560 /**
561  * e1000_rx_checksum - Receive Checksum Offload
562  * @adapter: board private structure
563  * @status_err: receive descriptor status and error fields
564  * @csum: receive descriptor csum field
565  * @sk_buff: socket buffer with received data
566  **/
567 static void e1000_rx_checksum(struct e1000_adapter *adapter, u32 status_err,
568                               struct sk_buff *skb)
569 {
570         u16 status = (u16)status_err;
571         u8 errors = (u8)(status_err >> 24);
572
573         skb_checksum_none_assert(skb);
574
575         /* Rx checksum disabled */
576         if (!(adapter->netdev->features & NETIF_F_RXCSUM))
577                 return;
578
579         /* Ignore Checksum bit is set */
580         if (status & E1000_RXD_STAT_IXSM)
581                 return;
582
583         /* TCP/UDP checksum error bit or IP checksum error bit is set */
584         if (errors & (E1000_RXD_ERR_TCPE | E1000_RXD_ERR_IPE)) {
585                 /* let the stack verify checksum errors */
586                 adapter->hw_csum_err++;
587                 return;
588         }
589
590         /* TCP/UDP Checksum has not been calculated */
591         if (!(status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)))
592                 return;
593
594         /* It must be a TCP or UDP packet with a valid checksum */
595         skb->ip_summed = CHECKSUM_UNNECESSARY;
596         adapter->hw_csum_good++;
597 }
598
599 static void e1000e_update_rdt_wa(struct e1000_ring *rx_ring, unsigned int i)
600 {
601         struct e1000_adapter *adapter = rx_ring->adapter;
602         struct e1000_hw *hw = &adapter->hw;
603         s32 ret_val = __ew32_prepare(hw);
604
605         writel(i, rx_ring->tail);
606
607         if (unlikely(!ret_val && (i != readl(rx_ring->tail)))) {
608                 u32 rctl = er32(RCTL);
609                 ew32(RCTL, rctl & ~E1000_RCTL_EN);
610                 e_err("ME firmware caused invalid RDT - resetting\n");
611                 schedule_work(&adapter->reset_task);
612         }
613 }
614
615 static void e1000e_update_tdt_wa(struct e1000_ring *tx_ring, unsigned int i)
616 {
617         struct e1000_adapter *adapter = tx_ring->adapter;
618         struct e1000_hw *hw = &adapter->hw;
619         s32 ret_val = __ew32_prepare(hw);
620
621         writel(i, tx_ring->tail);
622
623         if (unlikely(!ret_val && (i != readl(tx_ring->tail)))) {
624                 u32 tctl = er32(TCTL);
625                 ew32(TCTL, tctl & ~E1000_TCTL_EN);
626                 e_err("ME firmware caused invalid TDT - resetting\n");
627                 schedule_work(&adapter->reset_task);
628         }
629 }
630
631 /**
632  * e1000_alloc_rx_buffers - Replace used receive buffers
633  * @rx_ring: Rx descriptor ring
634  **/
635 static void e1000_alloc_rx_buffers(struct e1000_ring *rx_ring,
636                                    int cleaned_count, gfp_t gfp)
637 {
638         struct e1000_adapter *adapter = rx_ring->adapter;
639         struct net_device *netdev = adapter->netdev;
640         struct pci_dev *pdev = adapter->pdev;
641         union e1000_rx_desc_extended *rx_desc;
642         struct e1000_buffer *buffer_info;
643         struct sk_buff *skb;
644         unsigned int i;
645         unsigned int bufsz = adapter->rx_buffer_len;
646
647         i = rx_ring->next_to_use;
648         buffer_info = &rx_ring->buffer_info[i];
649
650         while (cleaned_count--) {
651                 skb = buffer_info->skb;
652                 if (skb) {
653                         skb_trim(skb, 0);
654                         goto map_skb;
655                 }
656
657                 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);
658                 if (!skb) {
659                         /* Better luck next round */
660                         adapter->alloc_rx_buff_failed++;
661                         break;
662                 }
663
664                 buffer_info->skb = skb;
665 map_skb:
666                 buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
667                                                   adapter->rx_buffer_len,
668                                                   DMA_FROM_DEVICE);
669                 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
670                         dev_err(&pdev->dev, "Rx DMA map failed\n");
671                         adapter->rx_dma_failed++;
672                         break;
673                 }
674
675                 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
676                 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
677
678                 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
679                         /* Force memory writes to complete before letting h/w
680                          * know there are new descriptors to fetch.  (Only
681                          * applicable for weak-ordered memory model archs,
682                          * such as IA-64).
683                          */
684                         wmb();
685                         if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
686                                 e1000e_update_rdt_wa(rx_ring, i);
687                         else
688                                 writel(i, rx_ring->tail);
689                 }
690                 i++;
691                 if (i == rx_ring->count)
692                         i = 0;
693                 buffer_info = &rx_ring->buffer_info[i];
694         }
695
696         rx_ring->next_to_use = i;
697 }
698
699 /**
700  * e1000_alloc_rx_buffers_ps - Replace used receive buffers; packet split
701  * @rx_ring: Rx descriptor ring
702  **/
703 static void e1000_alloc_rx_buffers_ps(struct e1000_ring *rx_ring,
704                                       int cleaned_count, gfp_t gfp)
705 {
706         struct e1000_adapter *adapter = rx_ring->adapter;
707         struct net_device *netdev = adapter->netdev;
708         struct pci_dev *pdev = adapter->pdev;
709         union e1000_rx_desc_packet_split *rx_desc;
710         struct e1000_buffer *buffer_info;
711         struct e1000_ps_page *ps_page;
712         struct sk_buff *skb;
713         unsigned int i, j;
714
715         i = rx_ring->next_to_use;
716         buffer_info = &rx_ring->buffer_info[i];
717
718         while (cleaned_count--) {
719                 rx_desc = E1000_RX_DESC_PS(*rx_ring, i);
720
721                 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
722                         ps_page = &buffer_info->ps_pages[j];
723                         if (j >= adapter->rx_ps_pages) {
724                                 /* all unused desc entries get hw null ptr */
725                                 rx_desc->read.buffer_addr[j + 1] =
726                                     ~cpu_to_le64(0);
727                                 continue;
728                         }
729                         if (!ps_page->page) {
730                                 ps_page->page = alloc_page(gfp);
731                                 if (!ps_page->page) {
732                                         adapter->alloc_rx_buff_failed++;
733                                         goto no_buffers;
734                                 }
735                                 ps_page->dma = dma_map_page(&pdev->dev,
736                                                             ps_page->page,
737                                                             0, PAGE_SIZE,
738                                                             DMA_FROM_DEVICE);
739                                 if (dma_mapping_error(&pdev->dev,
740                                                       ps_page->dma)) {
741                                         dev_err(&adapter->pdev->dev,
742                                                 "Rx DMA page map failed\n");
743                                         adapter->rx_dma_failed++;
744                                         goto no_buffers;
745                                 }
746                         }
747                         /* Refresh the desc even if buffer_addrs
748                          * didn't change because each write-back
749                          * erases this info.
750                          */
751                         rx_desc->read.buffer_addr[j + 1] =
752                             cpu_to_le64(ps_page->dma);
753                 }
754
755                 skb = __netdev_alloc_skb_ip_align(netdev, adapter->rx_ps_bsize0,
756                                                   gfp);
757
758                 if (!skb) {
759                         adapter->alloc_rx_buff_failed++;
760                         break;
761                 }
762
763                 buffer_info->skb = skb;
764                 buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
765                                                   adapter->rx_ps_bsize0,
766                                                   DMA_FROM_DEVICE);
767                 if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
768                         dev_err(&pdev->dev, "Rx DMA map failed\n");
769                         adapter->rx_dma_failed++;
770                         /* cleanup skb */
771                         dev_kfree_skb_any(skb);
772                         buffer_info->skb = NULL;
773                         break;
774                 }
775
776                 rx_desc->read.buffer_addr[0] = cpu_to_le64(buffer_info->dma);
777
778                 if (unlikely(!(i & (E1000_RX_BUFFER_WRITE - 1)))) {
779                         /* Force memory writes to complete before letting h/w
780                          * know there are new descriptors to fetch.  (Only
781                          * applicable for weak-ordered memory model archs,
782                          * such as IA-64).
783                          */
784                         wmb();
785                         if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
786                                 e1000e_update_rdt_wa(rx_ring, i << 1);
787                         else
788                                 writel(i << 1, rx_ring->tail);
789                 }
790
791                 i++;
792                 if (i == rx_ring->count)
793                         i = 0;
794                 buffer_info = &rx_ring->buffer_info[i];
795         }
796
797 no_buffers:
798         rx_ring->next_to_use = i;
799 }
800
801 /**
802  * e1000_alloc_jumbo_rx_buffers - Replace used jumbo receive buffers
803  * @rx_ring: Rx descriptor ring
804  * @cleaned_count: number of buffers to allocate this pass
805  **/
806
807 static void e1000_alloc_jumbo_rx_buffers(struct e1000_ring *rx_ring,
808                                          int cleaned_count, gfp_t gfp)
809 {
810         struct e1000_adapter *adapter = rx_ring->adapter;
811         struct net_device *netdev = adapter->netdev;
812         struct pci_dev *pdev = adapter->pdev;
813         union e1000_rx_desc_extended *rx_desc;
814         struct e1000_buffer *buffer_info;
815         struct sk_buff *skb;
816         unsigned int i;
817         unsigned int bufsz = 256 - 16;  /* for skb_reserve */
818
819         i = rx_ring->next_to_use;
820         buffer_info = &rx_ring->buffer_info[i];
821
822         while (cleaned_count--) {
823                 skb = buffer_info->skb;
824                 if (skb) {
825                         skb_trim(skb, 0);
826                         goto check_page;
827                 }
828
829                 skb = __netdev_alloc_skb_ip_align(netdev, bufsz, gfp);
830                 if (unlikely(!skb)) {
831                         /* Better luck next round */
832                         adapter->alloc_rx_buff_failed++;
833                         break;
834                 }
835
836                 buffer_info->skb = skb;
837 check_page:
838                 /* allocate a new page if necessary */
839                 if (!buffer_info->page) {
840                         buffer_info->page = alloc_page(gfp);
841                         if (unlikely(!buffer_info->page)) {
842                                 adapter->alloc_rx_buff_failed++;
843                                 break;
844                         }
845                 }
846
847                 if (!buffer_info->dma) {
848                         buffer_info->dma = dma_map_page(&pdev->dev,
849                                                         buffer_info->page, 0,
850                                                         PAGE_SIZE,
851                                                         DMA_FROM_DEVICE);
852                         if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
853                                 adapter->alloc_rx_buff_failed++;
854                                 break;
855                         }
856                 }
857
858                 rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
859                 rx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
860
861                 if (unlikely(++i == rx_ring->count))
862                         i = 0;
863                 buffer_info = &rx_ring->buffer_info[i];
864         }
865
866         if (likely(rx_ring->next_to_use != i)) {
867                 rx_ring->next_to_use = i;
868                 if (unlikely(i-- == 0))
869                         i = (rx_ring->count - 1);
870
871                 /* Force memory writes to complete before letting h/w
872                  * know there are new descriptors to fetch.  (Only
873                  * applicable for weak-ordered memory model archs,
874                  * such as IA-64).
875                  */
876                 wmb();
877                 if (adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
878                         e1000e_update_rdt_wa(rx_ring, i);
879                 else
880                         writel(i, rx_ring->tail);
881         }
882 }
883
884 static inline void e1000_rx_hash(struct net_device *netdev, __le32 rss,
885                                  struct sk_buff *skb)
886 {
887         if (netdev->features & NETIF_F_RXHASH)
888                 skb->rxhash = le32_to_cpu(rss);
889 }
890
891 /**
892  * e1000_clean_rx_irq - Send received data up the network stack
893  * @rx_ring: Rx descriptor ring
894  *
895  * the return value indicates whether actual cleaning was done, there
896  * is no guarantee that everything was cleaned
897  **/
898 static bool e1000_clean_rx_irq(struct e1000_ring *rx_ring, int *work_done,
899                                int work_to_do)
900 {
901         struct e1000_adapter *adapter = rx_ring->adapter;
902         struct net_device *netdev = adapter->netdev;
903         struct pci_dev *pdev = adapter->pdev;
904         struct e1000_hw *hw = &adapter->hw;
905         union e1000_rx_desc_extended *rx_desc, *next_rxd;
906         struct e1000_buffer *buffer_info, *next_buffer;
907         u32 length, staterr;
908         unsigned int i;
909         int cleaned_count = 0;
910         bool cleaned = false;
911         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
912
913         i = rx_ring->next_to_clean;
914         rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
915         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
916         buffer_info = &rx_ring->buffer_info[i];
917
918         while (staterr & E1000_RXD_STAT_DD) {
919                 struct sk_buff *skb;
920
921                 if (*work_done >= work_to_do)
922                         break;
923                 (*work_done)++;
924                 rmb();  /* read descriptor and rx_buffer_info after status DD */
925
926                 skb = buffer_info->skb;
927                 buffer_info->skb = NULL;
928
929                 prefetch(skb->data - NET_IP_ALIGN);
930
931                 i++;
932                 if (i == rx_ring->count)
933                         i = 0;
934                 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i);
935                 prefetch(next_rxd);
936
937                 next_buffer = &rx_ring->buffer_info[i];
938
939                 cleaned = true;
940                 cleaned_count++;
941                 dma_unmap_single(&pdev->dev, buffer_info->dma,
942                                  adapter->rx_buffer_len, DMA_FROM_DEVICE);
943                 buffer_info->dma = 0;
944
945                 length = le16_to_cpu(rx_desc->wb.upper.length);
946
947                 /* !EOP means multiple descriptors were used to store a single
948                  * packet, if that's the case we need to toss it.  In fact, we
949                  * need to toss every packet with the EOP bit clear and the
950                  * next frame that _does_ have the EOP bit set, as it is by
951                  * definition only a frame fragment
952                  */
953                 if (unlikely(!(staterr & E1000_RXD_STAT_EOP)))
954                         adapter->flags2 |= FLAG2_IS_DISCARDING;
955
956                 if (adapter->flags2 & FLAG2_IS_DISCARDING) {
957                         /* All receives must fit into a single buffer */
958                         e_dbg("Receive packet consumed multiple buffers\n");
959                         /* recycle */
960                         buffer_info->skb = skb;
961                         if (staterr & E1000_RXD_STAT_EOP)
962                                 adapter->flags2 &= ~FLAG2_IS_DISCARDING;
963                         goto next_desc;
964                 }
965
966                 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
967                              !(netdev->features & NETIF_F_RXALL))) {
968                         /* recycle */
969                         buffer_info->skb = skb;
970                         goto next_desc;
971                 }
972
973                 /* adjust length to remove Ethernet CRC */
974                 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
975                         /* If configured to store CRC, don't subtract FCS,
976                          * but keep the FCS bytes out of the total_rx_bytes
977                          * counter
978                          */
979                         if (netdev->features & NETIF_F_RXFCS)
980                                 total_rx_bytes -= 4;
981                         else
982                                 length -= 4;
983                 }
984
985                 total_rx_bytes += length;
986                 total_rx_packets++;
987
988                 /* code added for copybreak, this should improve
989                  * performance for small packets with large amounts
990                  * of reassembly being done in the stack
991                  */
992                 if (length < copybreak) {
993                         struct sk_buff *new_skb =
994                             netdev_alloc_skb_ip_align(netdev, length);
995                         if (new_skb) {
996                                 skb_copy_to_linear_data_offset(new_skb,
997                                                                -NET_IP_ALIGN,
998                                                                (skb->data -
999                                                                 NET_IP_ALIGN),
1000                                                                (length +
1001                                                                 NET_IP_ALIGN));
1002                                 /* save the skb in buffer_info as good */
1003                                 buffer_info->skb = skb;
1004                                 skb = new_skb;
1005                         }
1006                         /* else just continue with the old one */
1007                 }
1008                 /* end copybreak code */
1009                 skb_put(skb, length);
1010
1011                 /* Receive Checksum Offload */
1012                 e1000_rx_checksum(adapter, staterr, skb);
1013
1014                 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);
1015
1016                 e1000_receive_skb(adapter, netdev, skb, staterr,
1017                                   rx_desc->wb.upper.vlan);
1018
1019 next_desc:
1020                 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF);
1021
1022                 /* return some buffers to hardware, one at a time is too slow */
1023                 if (cleaned_count >= E1000_RX_BUFFER_WRITE) {
1024                         adapter->alloc_rx_buf(rx_ring, cleaned_count,
1025                                               GFP_ATOMIC);
1026                         cleaned_count = 0;
1027                 }
1028
1029                 /* use prefetched values */
1030                 rx_desc = next_rxd;
1031                 buffer_info = next_buffer;
1032
1033                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1034         }
1035         rx_ring->next_to_clean = i;
1036
1037         cleaned_count = e1000_desc_unused(rx_ring);
1038         if (cleaned_count)
1039                 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
1040
1041         adapter->total_rx_bytes += total_rx_bytes;
1042         adapter->total_rx_packets += total_rx_packets;
1043         return cleaned;
1044 }
1045
1046 static void e1000_put_txbuf(struct e1000_ring *tx_ring,
1047                             struct e1000_buffer *buffer_info)
1048 {
1049         struct e1000_adapter *adapter = tx_ring->adapter;
1050
1051         if (buffer_info->dma) {
1052                 if (buffer_info->mapped_as_page)
1053                         dma_unmap_page(&adapter->pdev->dev, buffer_info->dma,
1054                                        buffer_info->length, DMA_TO_DEVICE);
1055                 else
1056                         dma_unmap_single(&adapter->pdev->dev, buffer_info->dma,
1057                                          buffer_info->length, DMA_TO_DEVICE);
1058                 buffer_info->dma = 0;
1059         }
1060         if (buffer_info->skb) {
1061                 dev_kfree_skb_any(buffer_info->skb);
1062                 buffer_info->skb = NULL;
1063         }
1064         buffer_info->time_stamp = 0;
1065 }
1066
1067 static void e1000_print_hw_hang(struct work_struct *work)
1068 {
1069         struct e1000_adapter *adapter = container_of(work,
1070                                                      struct e1000_adapter,
1071                                                      print_hang_task);
1072         struct net_device *netdev = adapter->netdev;
1073         struct e1000_ring *tx_ring = adapter->tx_ring;
1074         unsigned int i = tx_ring->next_to_clean;
1075         unsigned int eop = tx_ring->buffer_info[i].next_to_watch;
1076         struct e1000_tx_desc *eop_desc = E1000_TX_DESC(*tx_ring, eop);
1077         struct e1000_hw *hw = &adapter->hw;
1078         u16 phy_status, phy_1000t_status, phy_ext_status;
1079         u16 pci_status;
1080
1081         if (test_bit(__E1000_DOWN, &adapter->state))
1082                 return;
1083
1084         if (!adapter->tx_hang_recheck && (adapter->flags2 & FLAG2_DMA_BURST)) {
1085                 /* May be block on write-back, flush and detect again
1086                  * flush pending descriptor writebacks to memory
1087                  */
1088                 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
1089                 /* execute the writes immediately */
1090                 e1e_flush();
1091                 /* Due to rare timing issues, write to TIDV again to ensure
1092                  * the write is successful
1093                  */
1094                 ew32(TIDV, adapter->tx_int_delay | E1000_TIDV_FPD);
1095                 /* execute the writes immediately */
1096                 e1e_flush();
1097                 adapter->tx_hang_recheck = true;
1098                 return;
1099         }
1100         /* Real hang detected */
1101         adapter->tx_hang_recheck = false;
1102         netif_stop_queue(netdev);
1103
1104         e1e_rphy(hw, MII_BMSR, &phy_status);
1105         e1e_rphy(hw, MII_STAT1000, &phy_1000t_status);
1106         e1e_rphy(hw, MII_ESTATUS, &phy_ext_status);
1107
1108         pci_read_config_word(adapter->pdev, PCI_STATUS, &pci_status);
1109
1110         /* detected Hardware unit hang */
1111         e_err("Detected Hardware Unit Hang:\n"
1112               "  TDH                  <%x>\n"
1113               "  TDT                  <%x>\n"
1114               "  next_to_use          <%x>\n"
1115               "  next_to_clean        <%x>\n"
1116               "buffer_info[next_to_clean]:\n"
1117               "  time_stamp           <%lx>\n"
1118               "  next_to_watch        <%x>\n"
1119               "  jiffies              <%lx>\n"
1120               "  next_to_watch.status <%x>\n"
1121               "MAC Status             <%x>\n"
1122               "PHY Status             <%x>\n"
1123               "PHY 1000BASE-T Status  <%x>\n"
1124               "PHY Extended Status    <%x>\n"
1125               "PCI Status             <%x>\n",
1126               readl(tx_ring->head), readl(tx_ring->tail), tx_ring->next_to_use,
1127               tx_ring->next_to_clean, tx_ring->buffer_info[eop].time_stamp,
1128               eop, jiffies, eop_desc->upper.fields.status, er32(STATUS),
1129               phy_status, phy_1000t_status, phy_ext_status, pci_status);
1130
1131         /* Suggest workaround for known h/w issue */
1132         if ((hw->mac.type == e1000_pchlan) && (er32(CTRL) & E1000_CTRL_TFCE))
1133                 e_err("Try turning off Tx pause (flow control) via ethtool\n");
1134 }
1135
1136 /**
1137  * e1000e_tx_hwtstamp_work - check for Tx time stamp
1138  * @work: pointer to work struct
1139  *
1140  * This work function polls the TSYNCTXCTL valid bit to determine when a
1141  * timestamp has been taken for the current stored skb.  The timestamp must
1142  * be for this skb because only one such packet is allowed in the queue.
1143  */
1144 static void e1000e_tx_hwtstamp_work(struct work_struct *work)
1145 {
1146         struct e1000_adapter *adapter = container_of(work, struct e1000_adapter,
1147                                                      tx_hwtstamp_work);
1148         struct e1000_hw *hw = &adapter->hw;
1149
1150         if (!adapter->tx_hwtstamp_skb)
1151                 return;
1152
1153         if (er32(TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID) {
1154                 struct skb_shared_hwtstamps shhwtstamps;
1155                 u64 txstmp;
1156
1157                 txstmp = er32(TXSTMPL);
1158                 txstmp |= (u64)er32(TXSTMPH) << 32;
1159
1160                 e1000e_systim_to_hwtstamp(adapter, &shhwtstamps, txstmp);
1161
1162                 skb_tstamp_tx(adapter->tx_hwtstamp_skb, &shhwtstamps);
1163                 dev_kfree_skb_any(adapter->tx_hwtstamp_skb);
1164                 adapter->tx_hwtstamp_skb = NULL;
1165         } else {
1166                 /* reschedule to check later */
1167                 schedule_work(&adapter->tx_hwtstamp_work);
1168         }
1169 }
1170
1171 /**
1172  * e1000_clean_tx_irq - Reclaim resources after transmit completes
1173  * @tx_ring: Tx descriptor ring
1174  *
1175  * the return value indicates whether actual cleaning was done, there
1176  * is no guarantee that everything was cleaned
1177  **/
1178 static bool e1000_clean_tx_irq(struct e1000_ring *tx_ring)
1179 {
1180         struct e1000_adapter *adapter = tx_ring->adapter;
1181         struct net_device *netdev = adapter->netdev;
1182         struct e1000_hw *hw = &adapter->hw;
1183         struct e1000_tx_desc *tx_desc, *eop_desc;
1184         struct e1000_buffer *buffer_info;
1185         unsigned int i, eop;
1186         unsigned int count = 0;
1187         unsigned int total_tx_bytes = 0, total_tx_packets = 0;
1188         unsigned int bytes_compl = 0, pkts_compl = 0;
1189
1190         i = tx_ring->next_to_clean;
1191         eop = tx_ring->buffer_info[i].next_to_watch;
1192         eop_desc = E1000_TX_DESC(*tx_ring, eop);
1193
1194         while ((eop_desc->upper.data & cpu_to_le32(E1000_TXD_STAT_DD)) &&
1195                (count < tx_ring->count)) {
1196                 bool cleaned = false;
1197                 rmb();          /* read buffer_info after eop_desc */
1198                 for (; !cleaned; count++) {
1199                         tx_desc = E1000_TX_DESC(*tx_ring, i);
1200                         buffer_info = &tx_ring->buffer_info[i];
1201                         cleaned = (i == eop);
1202
1203                         if (cleaned) {
1204                                 total_tx_packets += buffer_info->segs;
1205                                 total_tx_bytes += buffer_info->bytecount;
1206                                 if (buffer_info->skb) {
1207                                         bytes_compl += buffer_info->skb->len;
1208                                         pkts_compl++;
1209                                 }
1210                         }
1211
1212                         e1000_put_txbuf(tx_ring, buffer_info);
1213                         tx_desc->upper.data = 0;
1214
1215                         i++;
1216                         if (i == tx_ring->count)
1217                                 i = 0;
1218                 }
1219
1220                 if (i == tx_ring->next_to_use)
1221                         break;
1222                 eop = tx_ring->buffer_info[i].next_to_watch;
1223                 eop_desc = E1000_TX_DESC(*tx_ring, eop);
1224         }
1225
1226         tx_ring->next_to_clean = i;
1227
1228         netdev_completed_queue(netdev, pkts_compl, bytes_compl);
1229
1230 #define TX_WAKE_THRESHOLD 32
1231         if (count && netif_carrier_ok(netdev) &&
1232             e1000_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD) {
1233                 /* Make sure that anybody stopping the queue after this
1234                  * sees the new next_to_clean.
1235                  */
1236                 smp_mb();
1237
1238                 if (netif_queue_stopped(netdev) &&
1239                     !(test_bit(__E1000_DOWN, &adapter->state))) {
1240                         netif_wake_queue(netdev);
1241                         ++adapter->restart_queue;
1242                 }
1243         }
1244
1245         if (adapter->detect_tx_hung) {
1246                 /* Detect a transmit hang in hardware, this serializes the
1247                  * check with the clearing of time_stamp and movement of i
1248                  */
1249                 adapter->detect_tx_hung = false;
1250                 if (tx_ring->buffer_info[i].time_stamp &&
1251                     time_after(jiffies, tx_ring->buffer_info[i].time_stamp
1252                                + (adapter->tx_timeout_factor * HZ)) &&
1253                     !(er32(STATUS) & E1000_STATUS_TXOFF))
1254                         schedule_work(&adapter->print_hang_task);
1255                 else
1256                         adapter->tx_hang_recheck = false;
1257         }
1258         adapter->total_tx_bytes += total_tx_bytes;
1259         adapter->total_tx_packets += total_tx_packets;
1260         return count < tx_ring->count;
1261 }
1262
1263 /**
1264  * e1000_clean_rx_irq_ps - Send received data up the network stack; packet split
1265  * @rx_ring: Rx descriptor ring
1266  *
1267  * the return value indicates whether actual cleaning was done, there
1268  * is no guarantee that everything was cleaned
1269  **/
1270 static bool e1000_clean_rx_irq_ps(struct e1000_ring *rx_ring, int *work_done,
1271                                   int work_to_do)
1272 {
1273         struct e1000_adapter *adapter = rx_ring->adapter;
1274         struct e1000_hw *hw = &adapter->hw;
1275         union e1000_rx_desc_packet_split *rx_desc, *next_rxd;
1276         struct net_device *netdev = adapter->netdev;
1277         struct pci_dev *pdev = adapter->pdev;
1278         struct e1000_buffer *buffer_info, *next_buffer;
1279         struct e1000_ps_page *ps_page;
1280         struct sk_buff *skb;
1281         unsigned int i, j;
1282         u32 length, staterr;
1283         int cleaned_count = 0;
1284         bool cleaned = false;
1285         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1286
1287         i = rx_ring->next_to_clean;
1288         rx_desc = E1000_RX_DESC_PS(*rx_ring, i);
1289         staterr = le32_to_cpu(rx_desc->wb.middle.status_error);
1290         buffer_info = &rx_ring->buffer_info[i];
1291
1292         while (staterr & E1000_RXD_STAT_DD) {
1293                 if (*work_done >= work_to_do)
1294                         break;
1295                 (*work_done)++;
1296                 skb = buffer_info->skb;
1297                 rmb();  /* read descriptor and rx_buffer_info after status DD */
1298
1299                 /* in the packet split case this is header only */
1300                 prefetch(skb->data - NET_IP_ALIGN);
1301
1302                 i++;
1303                 if (i == rx_ring->count)
1304                         i = 0;
1305                 next_rxd = E1000_RX_DESC_PS(*rx_ring, i);
1306                 prefetch(next_rxd);
1307
1308                 next_buffer = &rx_ring->buffer_info[i];
1309
1310                 cleaned = true;
1311                 cleaned_count++;
1312                 dma_unmap_single(&pdev->dev, buffer_info->dma,
1313                                  adapter->rx_ps_bsize0, DMA_FROM_DEVICE);
1314                 buffer_info->dma = 0;
1315
1316                 /* see !EOP comment in other Rx routine */
1317                 if (!(staterr & E1000_RXD_STAT_EOP))
1318                         adapter->flags2 |= FLAG2_IS_DISCARDING;
1319
1320                 if (adapter->flags2 & FLAG2_IS_DISCARDING) {
1321                         e_dbg("Packet Split buffers didn't pick up the full packet\n");
1322                         dev_kfree_skb_irq(skb);
1323                         if (staterr & E1000_RXD_STAT_EOP)
1324                                 adapter->flags2 &= ~FLAG2_IS_DISCARDING;
1325                         goto next_desc;
1326                 }
1327
1328                 if (unlikely((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
1329                              !(netdev->features & NETIF_F_RXALL))) {
1330                         dev_kfree_skb_irq(skb);
1331                         goto next_desc;
1332                 }
1333
1334                 length = le16_to_cpu(rx_desc->wb.middle.length0);
1335
1336                 if (!length) {
1337                         e_dbg("Last part of the packet spanning multiple descriptors\n");
1338                         dev_kfree_skb_irq(skb);
1339                         goto next_desc;
1340                 }
1341
1342                 /* Good Receive */
1343                 skb_put(skb, length);
1344
1345                 {
1346                         /* this looks ugly, but it seems compiler issues make
1347                          * it more efficient than reusing j
1348                          */
1349                         int l1 = le16_to_cpu(rx_desc->wb.upper.length[0]);
1350
1351                         /* page alloc/put takes too long and effects small
1352                          * packet throughput, so unsplit small packets and
1353                          * save the alloc/put only valid in softirq (napi)
1354                          * context to call kmap_*
1355                          */
1356                         if (l1 && (l1 <= copybreak) &&
1357                             ((length + l1) <= adapter->rx_ps_bsize0)) {
1358                                 u8 *vaddr;
1359
1360                                 ps_page = &buffer_info->ps_pages[0];
1361
1362                                 /* there is no documentation about how to call
1363                                  * kmap_atomic, so we can't hold the mapping
1364                                  * very long
1365                                  */
1366                                 dma_sync_single_for_cpu(&pdev->dev,
1367                                                         ps_page->dma,
1368                                                         PAGE_SIZE,
1369                                                         DMA_FROM_DEVICE);
1370                                 vaddr = kmap_atomic(ps_page->page);
1371                                 memcpy(skb_tail_pointer(skb), vaddr, l1);
1372                                 kunmap_atomic(vaddr);
1373                                 dma_sync_single_for_device(&pdev->dev,
1374                                                            ps_page->dma,
1375                                                            PAGE_SIZE,
1376                                                            DMA_FROM_DEVICE);
1377
1378                                 /* remove the CRC */
1379                                 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
1380                                         if (!(netdev->features & NETIF_F_RXFCS))
1381                                                 l1 -= 4;
1382                                 }
1383
1384                                 skb_put(skb, l1);
1385                                 goto copydone;
1386                         }       /* if */
1387                 }
1388
1389                 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
1390                         length = le16_to_cpu(rx_desc->wb.upper.length[j]);
1391                         if (!length)
1392                                 break;
1393
1394                         ps_page = &buffer_info->ps_pages[j];
1395                         dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE,
1396                                        DMA_FROM_DEVICE);
1397                         ps_page->dma = 0;
1398                         skb_fill_page_desc(skb, j, ps_page->page, 0, length);
1399                         ps_page->page = NULL;
1400                         skb->len += length;
1401                         skb->data_len += length;
1402                         skb->truesize += PAGE_SIZE;
1403                 }
1404
1405                 /* strip the ethernet crc, problem is we're using pages now so
1406                  * this whole operation can get a little cpu intensive
1407                  */
1408                 if (!(adapter->flags2 & FLAG2_CRC_STRIPPING)) {
1409                         if (!(netdev->features & NETIF_F_RXFCS))
1410                                 pskb_trim(skb, skb->len - 4);
1411                 }
1412
1413 copydone:
1414                 total_rx_bytes += skb->len;
1415                 total_rx_packets++;
1416
1417                 e1000_rx_checksum(adapter, staterr, skb);
1418
1419                 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);
1420
1421                 if (rx_desc->wb.upper.header_status &
1422                     cpu_to_le16(E1000_RXDPS_HDRSTAT_HDRSP))
1423                         adapter->rx_hdr_split++;
1424
1425                 e1000_receive_skb(adapter, netdev, skb, staterr,
1426                                   rx_desc->wb.middle.vlan);
1427
1428 next_desc:
1429                 rx_desc->wb.middle.status_error &= cpu_to_le32(~0xFF);
1430                 buffer_info->skb = NULL;
1431
1432                 /* return some buffers to hardware, one at a time is too slow */
1433                 if (cleaned_count >= E1000_RX_BUFFER_WRITE) {
1434                         adapter->alloc_rx_buf(rx_ring, cleaned_count,
1435                                               GFP_ATOMIC);
1436                         cleaned_count = 0;
1437                 }
1438
1439                 /* use prefetched values */
1440                 rx_desc = next_rxd;
1441                 buffer_info = next_buffer;
1442
1443                 staterr = le32_to_cpu(rx_desc->wb.middle.status_error);
1444         }
1445         rx_ring->next_to_clean = i;
1446
1447         cleaned_count = e1000_desc_unused(rx_ring);
1448         if (cleaned_count)
1449                 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
1450
1451         adapter->total_rx_bytes += total_rx_bytes;
1452         adapter->total_rx_packets += total_rx_packets;
1453         return cleaned;
1454 }
1455
1456 /**
1457  * e1000_consume_page - helper function
1458  **/
1459 static void e1000_consume_page(struct e1000_buffer *bi, struct sk_buff *skb,
1460                                u16 length)
1461 {
1462         bi->page = NULL;
1463         skb->len += length;
1464         skb->data_len += length;
1465         skb->truesize += PAGE_SIZE;
1466 }
1467
1468 /**
1469  * e1000_clean_jumbo_rx_irq - Send received data up the network stack; legacy
1470  * @adapter: board private structure
1471  *
1472  * the return value indicates whether actual cleaning was done, there
1473  * is no guarantee that everything was cleaned
1474  **/
1475 static bool e1000_clean_jumbo_rx_irq(struct e1000_ring *rx_ring, int *work_done,
1476                                      int work_to_do)
1477 {
1478         struct e1000_adapter *adapter = rx_ring->adapter;
1479         struct net_device *netdev = adapter->netdev;
1480         struct pci_dev *pdev = adapter->pdev;
1481         union e1000_rx_desc_extended *rx_desc, *next_rxd;
1482         struct e1000_buffer *buffer_info, *next_buffer;
1483         u32 length, staterr;
1484         unsigned int i;
1485         int cleaned_count = 0;
1486         bool cleaned = false;
1487         unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1488         struct skb_shared_info *shinfo;
1489
1490         i = rx_ring->next_to_clean;
1491         rx_desc = E1000_RX_DESC_EXT(*rx_ring, i);
1492         staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1493         buffer_info = &rx_ring->buffer_info[i];
1494
1495         while (staterr & E1000_RXD_STAT_DD) {
1496                 struct sk_buff *skb;
1497
1498                 if (*work_done >= work_to_do)
1499                         break;
1500                 (*work_done)++;
1501                 rmb();  /* read descriptor and rx_buffer_info after status DD */
1502
1503                 skb = buffer_info->skb;
1504                 buffer_info->skb = NULL;
1505
1506                 ++i;
1507                 if (i == rx_ring->count)
1508                         i = 0;
1509                 next_rxd = E1000_RX_DESC_EXT(*rx_ring, i);
1510                 prefetch(next_rxd);
1511
1512                 next_buffer = &rx_ring->buffer_info[i];
1513
1514                 cleaned = true;
1515                 cleaned_count++;
1516                 dma_unmap_page(&pdev->dev, buffer_info->dma, PAGE_SIZE,
1517                                DMA_FROM_DEVICE);
1518                 buffer_info->dma = 0;
1519
1520                 length = le16_to_cpu(rx_desc->wb.upper.length);
1521
1522                 /* errors is only valid for DD + EOP descriptors */
1523                 if (unlikely((staterr & E1000_RXD_STAT_EOP) &&
1524                              ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) &&
1525                               !(netdev->features & NETIF_F_RXALL)))) {
1526                         /* recycle both page and skb */
1527                         buffer_info->skb = skb;
1528                         /* an error means any chain goes out the window too */
1529                         if (rx_ring->rx_skb_top)
1530                                 dev_kfree_skb_irq(rx_ring->rx_skb_top);
1531                         rx_ring->rx_skb_top = NULL;
1532                         goto next_desc;
1533                 }
1534 #define rxtop (rx_ring->rx_skb_top)
1535                 if (!(staterr & E1000_RXD_STAT_EOP)) {
1536                         /* this descriptor is only the beginning (or middle) */
1537                         if (!rxtop) {
1538                                 /* this is the beginning of a chain */
1539                                 rxtop = skb;
1540                                 skb_fill_page_desc(rxtop, 0, buffer_info->page,
1541                                                    0, length);
1542                         } else {
1543                                 /* this is the middle of a chain */
1544                                 shinfo = skb_shinfo(rxtop);
1545                                 skb_fill_page_desc(rxtop, shinfo->nr_frags,
1546                                                    buffer_info->page, 0,
1547                                                    length);
1548                                 /* re-use the skb, only consumed the page */
1549                                 buffer_info->skb = skb;
1550                         }
1551                         e1000_consume_page(buffer_info, rxtop, length);
1552                         goto next_desc;
1553                 } else {
1554                         if (rxtop) {
1555                                 /* end of the chain */
1556                                 shinfo = skb_shinfo(rxtop);
1557                                 skb_fill_page_desc(rxtop, shinfo->nr_frags,
1558                                                    buffer_info->page, 0,
1559                                                    length);
1560                                 /* re-use the current skb, we only consumed the
1561                                  * page
1562                                  */
1563                                 buffer_info->skb = skb;
1564                                 skb = rxtop;
1565                                 rxtop = NULL;
1566                                 e1000_consume_page(buffer_info, skb, length);
1567                         } else {
1568                                 /* no chain, got EOP, this buf is the packet
1569                                  * copybreak to save the put_page/alloc_page
1570                                  */
1571                                 if (length <= copybreak &&
1572                                     skb_tailroom(skb) >= length) {
1573                                         u8 *vaddr;
1574                                         vaddr = kmap_atomic(buffer_info->page);
1575                                         memcpy(skb_tail_pointer(skb), vaddr,
1576                                                length);
1577                                         kunmap_atomic(vaddr);
1578                                         /* re-use the page, so don't erase
1579                                          * buffer_info->page
1580                                          */
1581                                         skb_put(skb, length);
1582                                 } else {
1583                                         skb_fill_page_desc(skb, 0,
1584                                                            buffer_info->page, 0,
1585                                                            length);
1586                                         e1000_consume_page(buffer_info, skb,
1587                                                            length);
1588                                 }
1589                         }
1590                 }
1591
1592                 /* Receive Checksum Offload */
1593                 e1000_rx_checksum(adapter, staterr, skb);
1594
1595                 e1000_rx_hash(netdev, rx_desc->wb.lower.hi_dword.rss, skb);
1596
1597                 /* probably a little skewed due to removing CRC */
1598                 total_rx_bytes += skb->len;
1599                 total_rx_packets++;
1600
1601                 /* eth type trans needs skb->data to point to something */
1602                 if (!pskb_may_pull(skb, ETH_HLEN)) {
1603                         e_err("pskb_may_pull failed.\n");
1604                         dev_kfree_skb_irq(skb);
1605                         goto next_desc;
1606                 }
1607
1608                 e1000_receive_skb(adapter, netdev, skb, staterr,
1609                                   rx_desc->wb.upper.vlan);
1610
1611 next_desc:
1612                 rx_desc->wb.upper.status_error &= cpu_to_le32(~0xFF);
1613
1614                 /* return some buffers to hardware, one at a time is too slow */
1615                 if (unlikely(cleaned_count >= E1000_RX_BUFFER_WRITE)) {
1616                         adapter->alloc_rx_buf(rx_ring, cleaned_count,
1617                                               GFP_ATOMIC);
1618                         cleaned_count = 0;
1619                 }
1620
1621                 /* use prefetched values */
1622                 rx_desc = next_rxd;
1623                 buffer_info = next_buffer;
1624
1625                 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
1626         }
1627         rx_ring->next_to_clean = i;
1628
1629         cleaned_count = e1000_desc_unused(rx_ring);
1630         if (cleaned_count)
1631                 adapter->alloc_rx_buf(rx_ring, cleaned_count, GFP_ATOMIC);
1632
1633         adapter->total_rx_bytes += total_rx_bytes;
1634         adapter->total_rx_packets += total_rx_packets;
1635         return cleaned;
1636 }
1637
1638 /**
1639  * e1000_clean_rx_ring - Free Rx Buffers per Queue
1640  * @rx_ring: Rx descriptor ring
1641  **/
1642 static void e1000_clean_rx_ring(struct e1000_ring *rx_ring)
1643 {
1644         struct e1000_adapter *adapter = rx_ring->adapter;
1645         struct e1000_buffer *buffer_info;
1646         struct e1000_ps_page *ps_page;
1647         struct pci_dev *pdev = adapter->pdev;
1648         unsigned int i, j;
1649
1650         /* Free all the Rx ring sk_buffs */
1651         for (i = 0; i < rx_ring->count; i++) {
1652                 buffer_info = &rx_ring->buffer_info[i];
1653                 if (buffer_info->dma) {
1654                         if (adapter->clean_rx == e1000_clean_rx_irq)
1655                                 dma_unmap_single(&pdev->dev, buffer_info->dma,
1656                                                  adapter->rx_buffer_len,
1657                                                  DMA_FROM_DEVICE);
1658                         else if (adapter->clean_rx == e1000_clean_jumbo_rx_irq)
1659                                 dma_unmap_page(&pdev->dev, buffer_info->dma,
1660                                                PAGE_SIZE, DMA_FROM_DEVICE);
1661                         else if (adapter->clean_rx == e1000_clean_rx_irq_ps)
1662                                 dma_unmap_single(&pdev->dev, buffer_info->dma,
1663                                                  adapter->rx_ps_bsize0,
1664                                                  DMA_FROM_DEVICE);
1665                         buffer_info->dma = 0;
1666                 }
1667
1668                 if (buffer_info->page) {
1669                         put_page(buffer_info->page);
1670                         buffer_info->page = NULL;
1671                 }
1672
1673                 if (buffer_info->skb) {
1674                         dev_kfree_skb(buffer_info->skb);
1675                         buffer_info->skb = NULL;
1676                 }
1677
1678                 for (j = 0; j < PS_PAGE_BUFFERS; j++) {
1679                         ps_page = &buffer_info->ps_pages[j];
1680                         if (!ps_page->page)
1681                                 break;
1682                         dma_unmap_page(&pdev->dev, ps_page->dma, PAGE_SIZE,
1683                                        DMA_FROM_DEVICE);
1684                         ps_page->dma = 0;
1685                         put_page(ps_page->page);
1686                         ps_page->page = NULL;
1687                 }
1688         }
1689
1690         /* there also may be some cached data from a chained receive */
1691         if (rx_ring->rx_skb_top) {
1692                 dev_kfree_skb(rx_ring->rx_skb_top);
1693                 rx_ring->rx_skb_top = NULL;
1694         }
1695
1696         /* Zero out the descriptor ring */
1697         memset(rx_ring->desc, 0, rx_ring->size);
1698
1699         rx_ring->next_to_clean = 0;
1700         rx_ring->next_to_use = 0;
1701         adapter->flags2 &= ~FLAG2_IS_DISCARDING;
1702
1703         writel(0, rx_ring->head);
1704         if (rx_ring->adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
1705                 e1000e_update_rdt_wa(rx_ring, 0);
1706         else
1707                 writel(0, rx_ring->tail);
1708 }
1709
1710 static void e1000e_downshift_workaround(struct work_struct *work)
1711 {
1712         struct e1000_adapter *adapter = container_of(work,
1713                                                      struct e1000_adapter,
1714                                                      downshift_task);
1715
1716         if (test_bit(__E1000_DOWN, &adapter->state))
1717                 return;
1718
1719         e1000e_gig_downshift_workaround_ich8lan(&adapter->hw);
1720 }
1721
1722 /**
1723  * e1000_intr_msi - Interrupt Handler
1724  * @irq: interrupt number
1725  * @data: pointer to a network interface device structure
1726  **/
1727 static irqreturn_t e1000_intr_msi(int __always_unused irq, void *data)
1728 {
1729         struct net_device *netdev = data;
1730         struct e1000_adapter *adapter = netdev_priv(netdev);
1731         struct e1000_hw *hw = &adapter->hw;
1732         u32 icr = er32(ICR);
1733
1734         /* read ICR disables interrupts using IAM */
1735         if (icr & E1000_ICR_LSC) {
1736                 hw->mac.get_link_status = true;
1737                 /* ICH8 workaround-- Call gig speed drop workaround on cable
1738                  * disconnect (LSC) before accessing any PHY registers
1739                  */
1740                 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) &&
1741                     (!(er32(STATUS) & E1000_STATUS_LU)))
1742                         schedule_work(&adapter->downshift_task);
1743
1744                 /* 80003ES2LAN workaround-- For packet buffer work-around on
1745                  * link down event; disable receives here in the ISR and reset
1746                  * adapter in watchdog
1747                  */
1748                 if (netif_carrier_ok(netdev) &&
1749                     adapter->flags & FLAG_RX_NEEDS_RESTART) {
1750                         /* disable receives */
1751                         u32 rctl = er32(RCTL);
1752                         ew32(RCTL, rctl & ~E1000_RCTL_EN);
1753                         adapter->flags |= FLAG_RESTART_NOW;
1754                 }
1755                 /* guard against interrupt when we're going down */
1756                 if (!test_bit(__E1000_DOWN, &adapter->state))
1757                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
1758         }
1759
1760         /* Reset on uncorrectable ECC error */
1761         if ((icr & E1000_ICR_ECCER) && (hw->mac.type == e1000_pch_lpt)) {
1762                 u32 pbeccsts = er32(PBECCSTS);
1763
1764                 adapter->corr_errors +=
1765                     pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK;
1766                 adapter->uncorr_errors +=
1767                     (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >>
1768                     E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT;
1769
1770                 /* Do the reset outside of interrupt context */
1771                 schedule_work(&adapter->reset_task);
1772
1773                 /* return immediately since reset is imminent */
1774                 return IRQ_HANDLED;
1775         }
1776
1777         if (napi_schedule_prep(&adapter->napi)) {
1778                 adapter->total_tx_bytes = 0;
1779                 adapter->total_tx_packets = 0;
1780                 adapter->total_rx_bytes = 0;
1781                 adapter->total_rx_packets = 0;
1782                 __napi_schedule(&adapter->napi);
1783         }
1784
1785         return IRQ_HANDLED;
1786 }
1787
1788 /**
1789  * e1000_intr - Interrupt Handler
1790  * @irq: interrupt number
1791  * @data: pointer to a network interface device structure
1792  **/
1793 static irqreturn_t e1000_intr(int __always_unused irq, void *data)
1794 {
1795         struct net_device *netdev = data;
1796         struct e1000_adapter *adapter = netdev_priv(netdev);
1797         struct e1000_hw *hw = &adapter->hw;
1798         u32 rctl, icr = er32(ICR);
1799
1800         if (!icr || test_bit(__E1000_DOWN, &adapter->state))
1801                 return IRQ_NONE;        /* Not our interrupt */
1802
1803         /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
1804          * not set, then the adapter didn't send an interrupt
1805          */
1806         if (!(icr & E1000_ICR_INT_ASSERTED))
1807                 return IRQ_NONE;
1808
1809         /* Interrupt Auto-Mask...upon reading ICR,
1810          * interrupts are masked.  No need for the
1811          * IMC write
1812          */
1813
1814         if (icr & E1000_ICR_LSC) {
1815                 hw->mac.get_link_status = true;
1816                 /* ICH8 workaround-- Call gig speed drop workaround on cable
1817                  * disconnect (LSC) before accessing any PHY registers
1818                  */
1819                 if ((adapter->flags & FLAG_LSC_GIG_SPEED_DROP) &&
1820                     (!(er32(STATUS) & E1000_STATUS_LU)))
1821                         schedule_work(&adapter->downshift_task);
1822
1823                 /* 80003ES2LAN workaround--
1824                  * For packet buffer work-around on link down event;
1825                  * disable receives here in the ISR and
1826                  * reset adapter in watchdog
1827                  */
1828                 if (netif_carrier_ok(netdev) &&
1829                     (adapter->flags & FLAG_RX_NEEDS_RESTART)) {
1830                         /* disable receives */
1831                         rctl = er32(RCTL);
1832                         ew32(RCTL, rctl & ~E1000_RCTL_EN);
1833                         adapter->flags |= FLAG_RESTART_NOW;
1834                 }
1835                 /* guard against interrupt when we're going down */
1836                 if (!test_bit(__E1000_DOWN, &adapter->state))
1837                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
1838         }
1839
1840         /* Reset on uncorrectable ECC error */
1841         if ((icr & E1000_ICR_ECCER) && (hw->mac.type == e1000_pch_lpt)) {
1842                 u32 pbeccsts = er32(PBECCSTS);
1843
1844                 adapter->corr_errors +=
1845                     pbeccsts & E1000_PBECCSTS_CORR_ERR_CNT_MASK;
1846                 adapter->uncorr_errors +=
1847                     (pbeccsts & E1000_PBECCSTS_UNCORR_ERR_CNT_MASK) >>
1848                     E1000_PBECCSTS_UNCORR_ERR_CNT_SHIFT;
1849
1850                 /* Do the reset outside of interrupt context */
1851                 schedule_work(&adapter->reset_task);
1852
1853                 /* return immediately since reset is imminent */
1854                 return IRQ_HANDLED;
1855         }
1856
1857         if (napi_schedule_prep(&adapter->napi)) {
1858                 adapter->total_tx_bytes = 0;
1859                 adapter->total_tx_packets = 0;
1860                 adapter->total_rx_bytes = 0;
1861                 adapter->total_rx_packets = 0;
1862                 __napi_schedule(&adapter->napi);
1863         }
1864
1865         return IRQ_HANDLED;
1866 }
1867
1868 static irqreturn_t e1000_msix_other(int __always_unused irq, void *data)
1869 {
1870         struct net_device *netdev = data;
1871         struct e1000_adapter *adapter = netdev_priv(netdev);
1872         struct e1000_hw *hw = &adapter->hw;
1873         u32 icr = er32(ICR);
1874
1875         if (!(icr & E1000_ICR_INT_ASSERTED)) {
1876                 if (!test_bit(__E1000_DOWN, &adapter->state))
1877                         ew32(IMS, E1000_IMS_OTHER);
1878                 return IRQ_NONE;
1879         }
1880
1881         if (icr & adapter->eiac_mask)
1882                 ew32(ICS, (icr & adapter->eiac_mask));
1883
1884         if (icr & E1000_ICR_OTHER) {
1885                 if (!(icr & E1000_ICR_LSC))
1886                         goto no_link_interrupt;
1887                 hw->mac.get_link_status = true;
1888                 /* guard against interrupt when we're going down */
1889                 if (!test_bit(__E1000_DOWN, &adapter->state))
1890                         mod_timer(&adapter->watchdog_timer, jiffies + 1);
1891         }
1892
1893 no_link_interrupt:
1894         if (!test_bit(__E1000_DOWN, &adapter->state))
1895                 ew32(IMS, E1000_IMS_LSC | E1000_IMS_OTHER);
1896
1897         return IRQ_HANDLED;
1898 }
1899
1900 static irqreturn_t e1000_intr_msix_tx(int __always_unused irq, void *data)
1901 {
1902         struct net_device *netdev = data;
1903         struct e1000_adapter *adapter = netdev_priv(netdev);
1904         struct e1000_hw *hw = &adapter->hw;
1905         struct e1000_ring *tx_ring = adapter->tx_ring;
1906
1907         adapter->total_tx_bytes = 0;
1908         adapter->total_tx_packets = 0;
1909
1910         if (!e1000_clean_tx_irq(tx_ring))
1911                 /* Ring was not completely cleaned, so fire another interrupt */
1912                 ew32(ICS, tx_ring->ims_val);
1913
1914         return IRQ_HANDLED;
1915 }
1916
1917 static irqreturn_t e1000_intr_msix_rx(int __always_unused irq, void *data)
1918 {
1919         struct net_device *netdev = data;
1920         struct e1000_adapter *adapter = netdev_priv(netdev);
1921         struct e1000_ring *rx_ring = adapter->rx_ring;
1922
1923         /* Write the ITR value calculated at the end of the
1924          * previous interrupt.
1925          */
1926         if (rx_ring->set_itr) {
1927                 writel(1000000000 / (rx_ring->itr_val * 256),
1928                        rx_ring->itr_register);
1929                 rx_ring->set_itr = 0;
1930         }
1931
1932         if (napi_schedule_prep(&adapter->napi)) {
1933                 adapter->total_rx_bytes = 0;
1934                 adapter->total_rx_packets = 0;
1935                 __napi_schedule(&adapter->napi);
1936         }
1937         return IRQ_HANDLED;
1938 }
1939
1940 /**
1941  * e1000_configure_msix - Configure MSI-X hardware
1942  *
1943  * e1000_configure_msix sets up the hardware to properly
1944  * generate MSI-X interrupts.
1945  **/
1946 static void e1000_configure_msix(struct e1000_adapter *adapter)
1947 {
1948         struct e1000_hw *hw = &adapter->hw;
1949         struct e1000_ring *rx_ring = adapter->rx_ring;
1950         struct e1000_ring *tx_ring = adapter->tx_ring;
1951         int vector = 0;
1952         u32 ctrl_ext, ivar = 0;
1953
1954         adapter->eiac_mask = 0;
1955
1956         /* Workaround issue with spurious interrupts on 82574 in MSI-X mode */
1957         if (hw->mac.type == e1000_82574) {
1958                 u32 rfctl = er32(RFCTL);
1959                 rfctl |= E1000_RFCTL_ACK_DIS;
1960                 ew32(RFCTL, rfctl);
1961         }
1962
1963         /* Configure Rx vector */
1964         rx_ring->ims_val = E1000_IMS_RXQ0;
1965         adapter->eiac_mask |= rx_ring->ims_val;
1966         if (rx_ring->itr_val)
1967                 writel(1000000000 / (rx_ring->itr_val * 256),
1968                        rx_ring->itr_register);
1969         else
1970                 writel(1, rx_ring->itr_register);
1971         ivar = E1000_IVAR_INT_ALLOC_VALID | vector;
1972
1973         /* Configure Tx vector */
1974         tx_ring->ims_val = E1000_IMS_TXQ0;
1975         vector++;
1976         if (tx_ring->itr_val)
1977                 writel(1000000000 / (tx_ring->itr_val * 256),
1978                        tx_ring->itr_register);
1979         else
1980                 writel(1, tx_ring->itr_register);
1981         adapter->eiac_mask |= tx_ring->ims_val;
1982         ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 8);
1983
1984         /* set vector for Other Causes, e.g. link changes */
1985         vector++;
1986         ivar |= ((E1000_IVAR_INT_ALLOC_VALID | vector) << 16);
1987         if (rx_ring->itr_val)
1988                 writel(1000000000 / (rx_ring->itr_val * 256),
1989                        hw->hw_addr + E1000_EITR_82574(vector));
1990         else
1991                 writel(1, hw->hw_addr + E1000_EITR_82574(vector));
1992
1993         /* Cause Tx interrupts on every write back */
1994         ivar |= (1 << 31);
1995
1996         ew32(IVAR, ivar);
1997
1998         /* enable MSI-X PBA support */
1999         ctrl_ext = er32(CTRL_EXT);
2000         ctrl_ext |= E1000_CTRL_EXT_PBA_CLR;
2001
2002         /* Auto-Mask Other interrupts upon ICR read */
2003         ew32(IAM, ~E1000_EIAC_MASK_82574 | E1000_IMS_OTHER);
2004         ctrl_ext |= E1000_CTRL_EXT_EIAME;
2005         ew32(CTRL_EXT, ctrl_ext);
2006         e1e_flush();
2007 }
2008
2009 void e1000e_reset_interrupt_capability(struct e1000_adapter *adapter)
2010 {
2011         if (adapter->msix_entries) {
2012                 pci_disable_msix(adapter->pdev);
2013                 kfree(adapter->msix_entries);
2014                 adapter->msix_entries = NULL;
2015         } else if (adapter->flags & FLAG_MSI_ENABLED) {
2016                 pci_disable_msi(adapter->pdev);
2017                 adapter->flags &= ~FLAG_MSI_ENABLED;
2018         }
2019 }
2020
2021 /**
2022  * e1000e_set_interrupt_capability - set MSI or MSI-X if supported
2023  *
2024  * Attempt to configure interrupts using the best available
2025  * capabilities of the hardware and kernel.
2026  **/
2027 void e1000e_set_interrupt_capability(struct e1000_adapter *adapter)
2028 {
2029         int err;
2030         int i;
2031
2032         switch (adapter->int_mode) {
2033         case E1000E_INT_MODE_MSIX:
2034                 if (adapter->flags & FLAG_HAS_MSIX) {
2035                         adapter->num_vectors = 3; /* RxQ0, TxQ0 and other */
2036                         adapter->msix_entries = kcalloc(adapter->num_vectors,
2037                                                         sizeof(struct
2038                                                                msix_entry),
2039                                                         GFP_KERNEL);
2040                         if (adapter->msix_entries) {
2041                                 for (i = 0; i < adapter->num_vectors; i++)
2042                                         adapter->msix_entries[i].entry = i;
2043
2044                                 err = pci_enable_msix(adapter->pdev,
2045                                                       adapter->msix_entries,
2046                                                       adapter->num_vectors);
2047                                 if (err == 0)
2048                                         return;
2049                         }
2050                         /* MSI-X failed, so fall through and try MSI */
2051                         e_err("Failed to initialize MSI-X interrupts.  Falling back to MSI interrupts.\n");
2052                         e1000e_reset_interrupt_capability(adapter);
2053                 }
2054                 adapter->int_mode = E1000E_INT_MODE_MSI;
2055                 /* Fall through */
2056         case E1000E_INT_MODE_MSI:
2057                 if (!pci_enable_msi(adapter->pdev)) {
2058                         adapter->flags |= FLAG_MSI_ENABLED;
2059                 } else {
2060                         adapter->int_mode = E1000E_INT_MODE_LEGACY;
2061                         e_err("Failed to initialize MSI interrupts.  Falling back to legacy interrupts.\n");
2062                 }
2063                 /* Fall through */
2064         case E1000E_INT_MODE_LEGACY:
2065                 /* Don't do anything; this is the system default */
2066                 break;
2067         }
2068
2069         /* store the number of vectors being used */
2070         adapter->num_vectors = 1;
2071 }
2072
2073 /**
2074  * e1000_request_msix - Initialize MSI-X interrupts
2075  *
2076  * e1000_request_msix allocates MSI-X vectors and requests interrupts from the
2077  * kernel.
2078  **/
2079 static int e1000_request_msix(struct e1000_adapter *adapter)
2080 {
2081         struct net_device *netdev = adapter->netdev;
2082         int err = 0, vector = 0;
2083
2084         if (strlen(netdev->name) < (IFNAMSIZ - 5))
2085                 snprintf(adapter->rx_ring->name,
2086                          sizeof(adapter->rx_ring->name) - 1,
2087                          "%s-rx-0", netdev->name);
2088         else
2089                 memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ);
2090         err = request_irq(adapter->msix_entries[vector].vector,
2091                           e1000_intr_msix_rx, 0, adapter->rx_ring->name,
2092                           netdev);
2093         if (err)
2094                 return err;
2095         adapter->rx_ring->itr_register = adapter->hw.hw_addr +
2096             E1000_EITR_82574(vector);
2097         adapter->rx_ring->itr_val = adapter->itr;
2098         vector++;
2099
2100         if (strlen(netdev->name) < (IFNAMSIZ - 5))
2101                 snprintf(adapter->tx_ring->name,
2102                          sizeof(adapter->tx_ring->name) - 1,
2103                          "%s-tx-0", netdev->name);
2104         else
2105                 memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ);
2106         err = request_irq(adapter->msix_entries[vector].vector,
2107                           e1000_intr_msix_tx, 0, adapter->tx_ring->name,
2108                           netdev);
2109         if (err)
2110                 return err;
2111         adapter->tx_ring->itr_register = adapter->hw.hw_addr +
2112             E1000_EITR_82574(vector);
2113         adapter->tx_ring->itr_val = adapter->itr;
2114         vector++;
2115
2116         err = request_irq(adapter->msix_entries[vector].vector,
2117                           e1000_msix_other, 0, netdev->name, netdev);
2118         if (err)
2119                 return err;
2120
2121         e1000_configure_msix(adapter);
2122
2123         return 0;
2124 }
2125
2126 /**
2127  * e1000_request_irq - initialize interrupts
2128  *
2129  * Attempts to configure interrupts using the best available
2130  * capabilities of the hardware and kernel.
2131  **/
2132 static int e1000_request_irq(struct e1000_adapter *adapter)
2133 {
2134         struct net_device *netdev = adapter->netdev;
2135         int err;
2136
2137         if (adapter->msix_entries) {
2138                 err = e1000_request_msix(adapter);
2139                 if (!err)
2140                         return err;
2141                 /* fall back to MSI */
2142                 e1000e_reset_interrupt_capability(adapter);
2143                 adapter->int_mode = E1000E_INT_MODE_MSI;
2144                 e1000e_set_interrupt_capability(adapter);
2145         }
2146         if (adapter->flags & FLAG_MSI_ENABLED) {
2147                 err = request_irq(adapter->pdev->irq, e1000_intr_msi, 0,
2148                                   netdev->name, netdev);
2149                 if (!err)
2150                         return err;
2151
2152                 /* fall back to legacy interrupt */
2153                 e1000e_reset_interrupt_capability(adapter);
2154                 adapter->int_mode = E1000E_INT_MODE_LEGACY;
2155         }
2156
2157         err = request_irq(adapter->pdev->irq, e1000_intr, IRQF_SHARED,
2158                           netdev->name, netdev);
2159         if (err)
2160                 e_err("Unable to allocate interrupt, Error: %d\n", err);
2161
2162         return err;
2163 }
2164
2165 static void e1000_free_irq(struct e1000_adapter *adapter)
2166 {
2167         struct net_device *netdev = adapter->netdev;
2168
2169         if (adapter->msix_entries) {
2170                 int vector = 0;
2171
2172                 free_irq(adapter->msix_entries[vector].vector, netdev);
2173                 vector++;
2174
2175                 free_irq(adapter->msix_entries[vector].vector, netdev);
2176                 vector++;
2177
2178                 /* Other Causes interrupt vector */
2179                 free_irq(adapter->msix_entries[vector].vector, netdev);
2180                 return;
2181         }
2182
2183         free_irq(adapter->pdev->irq, netdev);
2184 }
2185
2186 /**
2187  * e1000_irq_disable - Mask off interrupt generation on the NIC
2188  **/
2189 static void e1000_irq_disable(struct e1000_adapter *adapter)
2190 {
2191         struct e1000_hw *hw = &adapter->hw;
2192
2193         ew32(IMC, ~0);
2194         if (adapter->msix_entries)
2195                 ew32(EIAC_82574, 0);
2196         e1e_flush();
2197
2198         if (adapter->msix_entries) {
2199                 int i;
2200                 for (i = 0; i < adapter->num_vectors; i++)
2201                         synchronize_irq(adapter->msix_entries[i].vector);
2202         } else {
2203                 synchronize_irq(adapter->pdev->irq);
2204         }
2205 }
2206
2207 /**
2208  * e1000_irq_enable - Enable default interrupt generation settings
2209  **/
2210 static void e1000_irq_enable(struct e1000_adapter *adapter)
2211 {
2212         struct e1000_hw *hw = &adapter->hw;
2213
2214         if (adapter->msix_entries) {
2215                 ew32(EIAC_82574, adapter->eiac_mask & E1000_EIAC_MASK_82574);
2216                 ew32(IMS, adapter->eiac_mask | E1000_IMS_OTHER | E1000_IMS_LSC);
2217         } else if (hw->mac.type == e1000_pch_lpt) {
2218                 ew32(IMS, IMS_ENABLE_MASK | E1000_IMS_ECCER);
2219         } else {
2220                 ew32(IMS, IMS_ENABLE_MASK);
2221         }
2222         e1e_flush();
2223 }
2224
2225 /**
2226  * e1000e_get_hw_control - get control of the h/w from f/w
2227  * @adapter: address of board private structure
2228  *
2229  * e1000e_get_hw_control sets {CTRL_EXT|SWSM}:DRV_LOAD bit.
2230  * For ASF and Pass Through versions of f/w this means that
2231  * the driver is loaded. For AMT version (only with 82573)
2232  * of the f/w this means that the network i/f is open.
2233  **/
2234 void e1000e_get_hw_control(struct e1000_adapter *adapter)
2235 {
2236         struct e1000_hw *hw = &adapter->hw;
2237         u32 ctrl_ext;
2238         u32 swsm;
2239
2240         /* Let firmware know the driver has taken over */
2241         if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) {
2242                 swsm = er32(SWSM);
2243                 ew32(SWSM, swsm | E1000_SWSM_DRV_LOAD);
2244         } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) {
2245                 ctrl_ext = er32(CTRL_EXT);
2246                 ew32(CTRL_EXT, ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
2247         }
2248 }
2249
2250 /**
2251  * e1000e_release_hw_control - release control of the h/w to f/w
2252  * @adapter: address of board private structure
2253  *
2254  * e1000e_release_hw_control resets {CTRL_EXT|SWSM}:DRV_LOAD bit.
2255  * For ASF and Pass Through versions of f/w this means that the
2256  * driver is no longer loaded. For AMT version (only with 82573) i
2257  * of the f/w this means that the network i/f is closed.
2258  *
2259  **/
2260 void e1000e_release_hw_control(struct e1000_adapter *adapter)
2261 {
2262         struct e1000_hw *hw = &adapter->hw;
2263         u32 ctrl_ext;
2264         u32 swsm;
2265
2266         /* Let firmware taken over control of h/w */
2267         if (adapter->flags & FLAG_HAS_SWSM_ON_LOAD) {
2268                 swsm = er32(SWSM);
2269                 ew32(SWSM, swsm & ~E1000_SWSM_DRV_LOAD);
2270         } else if (adapter->flags & FLAG_HAS_CTRLEXT_ON_LOAD) {
2271                 ctrl_ext = er32(CTRL_EXT);
2272                 ew32(CTRL_EXT, ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
2273         }
2274 }
2275
2276 /**
2277  * e1000_alloc_ring_dma - allocate memory for a ring structure
2278  **/
2279 static int e1000_alloc_ring_dma(struct e1000_adapter *adapter,
2280                                 struct e1000_ring *ring)
2281 {
2282         struct pci_dev *pdev = adapter->pdev;
2283
2284         ring->desc = dma_alloc_coherent(&pdev->dev, ring->size, &ring->dma,
2285                                         GFP_KERNEL);
2286         if (!ring->desc)
2287                 return -ENOMEM;
2288
2289         return 0;
2290 }
2291
2292 /**
2293  * e1000e_setup_tx_resources - allocate Tx resources (Descriptors)
2294  * @tx_ring: Tx descriptor ring
2295  *
2296  * Return 0 on success, negative on failure
2297  **/
2298 int e1000e_setup_tx_resources(struct e1000_ring *tx_ring)
2299 {
2300         struct e1000_adapter *adapter = tx_ring->adapter;
2301         int err = -ENOMEM, size;
2302
2303         size = sizeof(struct e1000_buffer) * tx_ring->count;
2304         tx_ring->buffer_info = vzalloc(size);
2305         if (!tx_ring->buffer_info)
2306                 goto err;
2307
2308         /* round up to nearest 4K */
2309         tx_ring->size = tx_ring->count * sizeof(struct e1000_tx_desc);
2310         tx_ring->size = ALIGN(tx_ring->size, 4096);
2311
2312         err = e1000_alloc_ring_dma(adapter, tx_ring);
2313         if (err)
2314                 goto err;
2315
2316         tx_ring->next_to_use = 0;
2317         tx_ring->next_to_clean = 0;
2318
2319         return 0;
2320 err:
2321         vfree(tx_ring->buffer_info);
2322         e_err("Unable to allocate memory for the transmit descriptor ring\n");
2323         return err;
2324 }
2325
2326 /**
2327  * e1000e_setup_rx_resources - allocate Rx resources (Descriptors)
2328  * @rx_ring: Rx descriptor ring
2329  *
2330  * Returns 0 on success, negative on failure
2331  **/
2332 int e1000e_setup_rx_resources(struct e1000_ring *rx_ring)
2333 {
2334         struct e1000_adapter *adapter = rx_ring->adapter;
2335         struct e1000_buffer *buffer_info;
2336         int i, size, desc_len, err = -ENOMEM;
2337
2338         size = sizeof(struct e1000_buffer) * rx_ring->count;
2339         rx_ring->buffer_info = vzalloc(size);
2340         if (!rx_ring->buffer_info)
2341                 goto err;
2342
2343         for (i = 0; i < rx_ring->count; i++) {
2344                 buffer_info = &rx_ring->buffer_info[i];
2345                 buffer_info->ps_pages = kcalloc(PS_PAGE_BUFFERS,
2346                                                 sizeof(struct e1000_ps_page),
2347                                                 GFP_KERNEL);
2348                 if (!buffer_info->ps_pages)
2349                         goto err_pages;
2350         }
2351
2352         desc_len = sizeof(union e1000_rx_desc_packet_split);
2353
2354         /* Round up to nearest 4K */
2355         rx_ring->size = rx_ring->count * desc_len;
2356         rx_ring->size = ALIGN(rx_ring->size, 4096);
2357
2358         err = e1000_alloc_ring_dma(adapter, rx_ring);
2359         if (err)
2360                 goto err_pages;
2361
2362         rx_ring->next_to_clean = 0;
2363         rx_ring->next_to_use = 0;
2364         rx_ring->rx_skb_top = NULL;
2365
2366         return 0;
2367
2368 err_pages:
2369         for (i = 0; i < rx_ring->count; i++) {
2370                 buffer_info = &rx_ring->buffer_info[i];
2371                 kfree(buffer_info->ps_pages);
2372         }
2373 err:
2374         vfree(rx_ring->buffer_info);
2375         e_err("Unable to allocate memory for the receive descriptor ring\n");
2376         return err;
2377 }
2378
2379 /**
2380  * e1000_clean_tx_ring - Free Tx Buffers
2381  * @tx_ring: Tx descriptor ring
2382  **/
2383 static void e1000_clean_tx_ring(struct e1000_ring *tx_ring)
2384 {
2385         struct e1000_adapter *adapter = tx_ring->adapter;
2386         struct e1000_buffer *buffer_info;
2387         unsigned long size;
2388         unsigned int i;
2389
2390         for (i = 0; i < tx_ring->count; i++) {
2391                 buffer_info = &tx_ring->buffer_info[i];
2392                 e1000_put_txbuf(tx_ring, buffer_info);
2393         }
2394
2395         netdev_reset_queue(adapter->netdev);
2396         size = sizeof(struct e1000_buffer) * tx_ring->count;
2397         memset(tx_ring->buffer_info, 0, size);
2398
2399         memset(tx_ring->desc, 0, tx_ring->size);
2400
2401         tx_ring->next_to_use = 0;
2402         tx_ring->next_to_clean = 0;
2403
2404         writel(0, tx_ring->head);
2405         if (tx_ring->adapter->flags2 & FLAG2_PCIM2PCI_ARBITER_WA)
2406                 e1000e_update_tdt_wa(tx_ring, 0);
2407         else
2408                 writel(0, tx_ring->tail);
2409 }
2410
2411 /**
2412  * e1000e_free_tx_resources - Free Tx Resources per Queue
2413  * @tx_ring: Tx descriptor ring
2414  *
2415  * Free all transmit software resources
2416  **/
2417 void e1000e_free_tx_resources(struct e1000_ring *tx_ring)
2418 {
2419         struct e1000_adapter *adapter = tx_ring->adapter;
2420         struct pci_dev *pdev = adapter->pdev;
2421
2422         e1000_clean_tx_ring(tx_ring);
2423
2424         vfree(tx_ring->buffer_info);
2425         tx_ring->buffer_info = NULL;
2426
2427         dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
2428                           tx_ring->dma);
2429         tx_ring->desc = NULL;
2430 }
2431
2432 /**
2433  * e1000e_free_rx_resources - Free Rx Resources
2434  * @rx_ring: Rx descriptor ring
2435  *
2436  * Free all receive software resources
2437  **/
2438 void e1000e_free_rx_resources(struct e1000_ring *rx_ring)
2439 {
2440         struct e1000_adapter *adapter = rx_ring->adapter;
2441         struct pci_dev *pdev = adapter->pdev;
2442         int i;
2443
2444         e1000_clean_rx_ring(rx_ring);
2445
2446         for (i = 0; i < rx_ring->count; i++)
2447                 kfree(rx_ring->buffer_info[i].ps_pages);
2448
2449         vfree(rx_ring->buffer_info);
2450         rx_ring->buffer_info = NULL;
2451
2452         dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
2453                           rx_ring->dma);
2454         rx_ring->desc = NULL;
2455 }
2456
2457 /**
2458  * e1000_update_itr - update the dynamic ITR value based on statistics
2459  * @adapter: pointer to adapter
2460  * @itr_setting: current adapter->itr
2461  * @packets: the number of packets during this measurement interval
2462  * @bytes: the number of bytes during this measurement interval
2463  *
2464  *      Stores a new ITR value based on packets and byte
2465  *      counts during the last interrupt.  The advantage of per interrupt
2466  *      computation is faster updates and more accurate ITR for the current
2467  *      traffic pattern.  Constants in this function were computed
2468  *      based on theoretical maximum wire speed and thresholds were set based
2469  *      on testing data as well as attempting to minimize response time
2470  *      while increasing bulk throughput.  This functionality is controlled
2471  *      by the InterruptThrottleRate module parameter.
2472  **/
2473 static unsigned int e1000_update_itr(u16 itr_setting, int packets, int bytes)
2474 {
2475         unsigned int retval = itr_setting;
2476
2477         if (packets == 0)
2478                 return itr_setting;
2479
2480         switch (itr_setting) {
2481         case lowest_latency:
2482                 /* handle TSO and jumbo frames */
2483                 if (bytes / packets > 8000)
2484                         retval = bulk_latency;
2485                 else if ((packets < 5) && (bytes > 512))
2486                         retval = low_latency;
2487                 break;
2488         case low_latency:       /* 50 usec aka 20000 ints/s */
2489                 if (bytes > 10000) {
2490                         /* this if handles the TSO accounting */
2491                         if (bytes / packets > 8000)
2492                                 retval = bulk_latency;
2493                         else if ((packets < 10) || ((bytes / packets) > 1200))
2494                                 retval = bulk_latency;
2495                         else if ((packets > 35))
2496                                 retval = lowest_latency;
2497                 } else if (bytes / packets > 2000) {
2498                         retval = bulk_latency;
2499                 } else if (packets <= 2 && bytes < 512) {
2500                         retval = lowest_latency;
2501                 }
2502                 break;
2503         case bulk_latency:      /* 250 usec aka 4000 ints/s */
2504                 if (bytes > 25000) {
2505                         if (packets > 35)
2506                                 retval = low_latency;
2507                 } else if (bytes < 6000) {
2508                         retval = low_latency;
2509                 }
2510                 break;
2511         }
2512
2513         return retval;
2514 }
2515
2516 static void e1000_set_itr(struct e1000_adapter *adapter)
2517 {
2518         u16 current_itr;
2519         u32 new_itr = adapter->itr;
2520
2521         /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
2522         if (adapter->link_speed != SPEED_1000) {
2523                 current_itr = 0;
2524                 new_itr = 4000;
2525                 goto set_itr_now;
2526         }
2527
2528         if (adapter->flags2 & FLAG2_DISABLE_AIM) {
2529                 new_itr = 0;
2530                 goto set_itr_now;
2531         }
2532
2533         adapter->tx_itr = e1000_update_itr(adapter->tx_itr,
2534                                            adapter->total_tx_packets,
2535                                            adapter->total_tx_bytes);
2536         /* conservative mode (itr 3) eliminates the lowest_latency setting */
2537         if (adapter->itr_setting == 3 && adapter->tx_itr == lowest_latency)
2538                 adapter->tx_itr = low_latency;
2539
2540         adapter->rx_itr = e1000_update_itr(adapter->rx_itr,
2541                                            adapter->total_rx_packets,
2542                                            adapter->total_rx_bytes);
2543         /* conservative mode (itr 3) eliminates the lowest_latency setting */
2544         if (adapter->itr_setting == 3 && adapter->rx_itr == lowest_latency)
2545                 adapter->rx_itr = low_latency;
2546
2547         current_itr = max(adapter->rx_itr, adapter->tx_itr);
2548
2549         /* counts and packets in update_itr are dependent on these numbers */
2550         switch (current_itr) {
2551         case lowest_latency:
2552                 new_itr = 70000;
2553                 break;
2554         case low_latency:
2555                 new_itr = 20000;        /* aka hwitr = ~200 */
2556                 break;
2557         case bulk_latency:
2558                 new_itr = 4000;
2559                 break;
2560         default:
2561                 break;
2562         }
2563
2564 set_itr_now:
2565         if (new_itr != adapter->itr) {
2566                 /* this attempts to bias the interrupt rate towards Bulk
2567                  * by adding intermediate steps when interrupt rate is
2568                  * increasing
2569                  */
2570                 new_itr = new_itr > adapter->itr ?
2571                     min(adapter->itr + (new_itr >> 2), new_itr) : new_itr;
2572                 adapter->itr = new_itr;
2573                 adapter->rx_ring->itr_val = new_itr;
2574                 if (adapter->msix_entries)
2575                         adapter->rx_ring->set_itr = 1;
2576                 else
2577                         e1000e_write_itr(adapter, new_itr);
2578         }
2579 }
2580
2581 /**
2582  * e1000e_write_itr - write the ITR value to the appropriate registers
2583  * @adapter: address of board private structure
2584  * @itr: new ITR value to program
2585  *
2586  * e1000e_write_itr determines if the adapter is in MSI-X mode
2587  * and, if so, writes the EITR registers with the ITR value.
2588  * Otherwise, it writes the ITR value into the ITR register.
2589  **/
2590 void e1000e_write_itr(struct e1000_adapter *adapter, u32 itr)
2591 {
2592         struct e1000_hw *hw = &adapter->hw;
2593         u32 new_itr = itr ? 1000000000 / (itr * 256) : 0;
2594
2595         if (adapter->msix_entries) {
2596                 int vector;
2597
2598                 for (vector = 0; vector < adapter->num_vectors; vector++)
2599                         writel(new_itr, hw->hw_addr + E1000_EITR_82574(vector));
2600         } else {
2601                 ew32(ITR, new_itr);
2602         }
2603 }
2604
2605 /**
2606  * e1000_alloc_queues - Allocate memory for all rings
2607  * @adapter: board private structure to initialize
2608  **/
2609 static int e1000_alloc_queues(struct e1000_adapter *adapter)
2610 {
2611         int size = sizeof(struct e1000_ring);
2612
2613         adapter->tx_ring = kzalloc(size, GFP_KERNEL);
2614         if (!adapter->tx_ring)
2615                 goto err;
2616         adapter->tx_ring->count = adapter->tx_ring_count;
2617         adapter->tx_ring->adapter = adapter;
2618
2619         adapter->rx_ring = kzalloc(size, GFP_KERNEL);
2620         if (!adapter->rx_ring)
2621                 goto err;
2622         adapter->rx_ring->count = adapter->rx_ring_count;
2623         adapter->rx_ring->adapter = adapter;
2624
2625         return 0;
2626 err:
2627         e_err("Unable to allocate memory for queues\n");
2628         kfree(adapter->rx_ring);
2629         kfree(adapter->tx_ring);
2630         return -ENOMEM;
2631 }
2632
2633 /**
2634  * e1000e_poll - NAPI Rx polling callback
2635  * @napi: struct associated with this polling callback
2636  * @weight: number of packets driver is allowed to process this poll
2637  **/
2638 static int e1000e_poll(struct napi_struct *napi, int weight)
2639 {
2640         struct e1000_adapter *adapter = container_of(napi, struct e1000_adapter,
2641                                                      napi);
2642         struct e1000_hw *hw = &adapter->hw;
2643         struct net_device *poll_dev = adapter->netdev;
2644         int tx_cleaned = 1, work_done = 0;
2645
2646         adapter = netdev_priv(poll_dev);
2647
2648         if (!adapter->msix_entries ||
2649             (adapter->rx_ring->ims_val & adapter->tx_ring->ims_val))
2650                 tx_cleaned = e1000_clean_tx_irq(adapter->tx_ring);
2651
2652         adapter->clean_rx(adapter->rx_ring, &work_done, weight);
2653
2654         if (!tx_cleaned)
2655                 work_done = weight;
2656
2657         /* If weight not fully consumed, exit the polling mode */
2658         if (work_done < weight) {
2659                 if (adapter->itr_setting & 3)
2660                         e1000_set_itr(adapter);
2661                 napi_complete(napi);
2662                 if (!test_bit(__E1000_DOWN, &adapter->state)) {
2663                         if (adapter->msix_entries)
2664                                 ew32(IMS, adapter->rx_ring->ims_val);
2665                         else
2666                                 e1000_irq_enable(adapter);
2667                 }
2668         }
2669
2670         return work_done;
2671 }
2672
2673 static int e1000_vlan_rx_add_vid(struct net_device *netdev,
2674                                  __always_unused __be16 proto, u16 vid)
2675 {
2676         struct e1000_adapter *adapter = netdev_priv(netdev);
2677         struct e1000_hw *hw = &adapter->hw;
2678         u32 vfta, index;
2679
2680         /* don't update vlan cookie if already programmed */
2681         if ((adapter->hw.mng_cookie.status &
2682              E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
2683             (vid == adapter->mng_vlan_id))
2684                 return 0;
2685
2686         /* add VID to filter table */
2687         if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2688                 index = (vid >> 5) & 0x7F;
2689                 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index);
2690                 vfta |= (1 << (vid & 0x1F));
2691                 hw->mac.ops.write_vfta(hw, index, vfta);
2692         }
2693
2694         set_bit(vid, adapter->active_vlans);
2695
2696         return 0;
2697 }
2698
2699 static int e1000_vlan_rx_kill_vid(struct net_device *netdev,
2700                                   __always_unused __be16 proto, u16 vid)
2701 {
2702         struct e1000_adapter *adapter = netdev_priv(netdev);
2703         struct e1000_hw *hw = &adapter->hw;
2704         u32 vfta, index;
2705
2706         if ((adapter->hw.mng_cookie.status &
2707              E1000_MNG_DHCP_COOKIE_STATUS_VLAN) &&
2708             (vid == adapter->mng_vlan_id)) {
2709                 /* release control to f/w */
2710                 e1000e_release_hw_control(adapter);
2711                 return 0;
2712         }
2713
2714         /* remove VID from filter table */
2715         if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2716                 index = (vid >> 5) & 0x7F;
2717                 vfta = E1000_READ_REG_ARRAY(hw, E1000_VFTA, index);
2718                 vfta &= ~(1 << (vid & 0x1F));
2719                 hw->mac.ops.write_vfta(hw, index, vfta);
2720         }
2721
2722         clear_bit(vid, adapter->active_vlans);
2723
2724         return 0;
2725 }
2726
2727 /**
2728  * e1000e_vlan_filter_disable - helper to disable hw VLAN filtering
2729  * @adapter: board private structure to initialize
2730  **/
2731 static void e1000e_vlan_filter_disable(struct e1000_adapter *adapter)
2732 {
2733         struct net_device *netdev = adapter->netdev;
2734         struct e1000_hw *hw = &adapter->hw;
2735         u32 rctl;
2736
2737         if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2738                 /* disable VLAN receive filtering */
2739                 rctl = er32(RCTL);
2740                 rctl &= ~(E1000_RCTL_VFE | E1000_RCTL_CFIEN);
2741                 ew32(RCTL, rctl);
2742
2743                 if (adapter->mng_vlan_id != (u16)E1000_MNG_VLAN_NONE) {
2744                         e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q),
2745                                                adapter->mng_vlan_id);
2746                         adapter->mng_vlan_id = E1000_MNG_VLAN_NONE;
2747                 }
2748         }
2749 }
2750
2751 /**
2752  * e1000e_vlan_filter_enable - helper to enable HW VLAN filtering
2753  * @adapter: board private structure to initialize
2754  **/
2755 static void e1000e_vlan_filter_enable(struct e1000_adapter *adapter)
2756 {
2757         struct e1000_hw *hw = &adapter->hw;
2758         u32 rctl;
2759
2760         if (adapter->flags & FLAG_HAS_HW_VLAN_FILTER) {
2761                 /* enable VLAN receive filtering */
2762                 rctl = er32(RCTL);
2763                 rctl |= E1000_RCTL_VFE;
2764                 rctl &= ~E1000_RCTL_CFIEN;
2765                 ew32(RCTL, rctl);
2766         }
2767 }
2768
2769 /**
2770  * e1000e_vlan_strip_enable - helper to disable HW VLAN stripping
2771  * @adapter: board private structure to initialize
2772  **/
2773 static void e1000e_vlan_strip_disable(struct e1000_adapter *adapter)
2774 {
2775         struct e1000_hw *hw = &adapter->hw;
2776         u32 ctrl;
2777
2778         /* disable VLAN tag insert/strip */
2779         ctrl = er32(CTRL);
2780         ctrl &= ~E1000_CTRL_VME;
2781         ew32(CTRL, ctrl);
2782 }
2783
2784 /**
2785  * e1000e_vlan_strip_enable - helper to enable HW VLAN stripping
2786  * @adapter: board private structure to initialize
2787  **/
2788 static void e1000e_vlan_strip_enable(struct e1000_adapter *adapter)
2789 {
2790         struct e1000_hw *hw = &adapter->hw;
2791         u32 ctrl;
2792
2793         /* enable VLAN tag insert/strip */
2794         ctrl = er32(CTRL);
2795         ctrl |= E1000_CTRL_VME;
2796         ew32(CTRL, ctrl);
2797 }
2798
2799 static void e1000_update_mng_vlan(struct e1000_adapter *adapter)
2800 {
2801         struct net_device *netdev = adapter->netdev;
2802         u16 vid = adapter->hw.mng_cookie.vlan_id;
2803         u16 old_vid = adapter->mng_vlan_id;
2804
2805         if (adapter->hw.mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
2806                 e1000_vlan_rx_add_vid(netdev, htons(ETH_P_8021Q), vid);
2807                 adapter->mng_vlan_id = vid;
2808         }
2809
2810         if ((old_vid != (u16)E1000_MNG_VLAN_NONE) && (vid != old_vid))
2811                 e1000_vlan_rx_kill_vid(netdev, htons(ETH_P_8021Q), old_vid);
2812 }
2813
2814 static void e1000_restore_vlan(struct e1000_adapter *adapter)
2815 {
2816         u16 vid;
2817
2818         e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0);
2819
2820         for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
2821             e1000_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
2822 }
2823
2824 static void e1000_init_manageability_pt(struct e1000_adapter *adapter)
2825 {
2826         struct e1000_hw *hw = &adapter->hw;
2827         u32 manc, manc2h, mdef, i, j;
2828
2829         if (!(adapter->flags & FLAG_MNG_PT_ENABLED))
2830                 return;
2831
2832         manc = er32(MANC);
2833
2834         /* enable receiving management packets to the host. this will probably
2835          * generate destination unreachable messages from the host OS, but
2836          * the packets will be handled on SMBUS
2837          */
2838         manc |= E1000_MANC_EN_MNG2HOST;
2839         manc2h = er32(MANC2H);
2840
2841         switch (hw->mac.type) {
2842         default:
2843                 manc2h |= (E1000_MANC2H_PORT_623 | E1000_MANC2H_PORT_664);
2844                 break;
2845         case e1000_82574:
2846         case e1000_82583:
2847                 /* Check if IPMI pass-through decision filter already exists;
2848                  * if so, enable it.
2849                  */
2850                 for (i = 0, j = 0; i < 8; i++) {
2851                         mdef = er32(MDEF(i));
2852
2853                         /* Ignore filters with anything other than IPMI ports */
2854                         if (mdef & ~(E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664))
2855                                 continue;
2856
2857                         /* Enable this decision filter in MANC2H */
2858                         if (mdef)
2859                                 manc2h |= (1 << i);
2860
2861                         j |= mdef;
2862                 }
2863
2864                 if (j == (E1000_MDEF_PORT_623 | E1000_MDEF_PORT_664))
2865                         break;
2866
2867                 /* Create new decision filter in an empty filter */
2868                 for (i = 0, j = 0; i < 8; i++)
2869                         if (er32(MDEF(i)) == 0) {
2870                                 ew32(MDEF(i), (E1000_MDEF_PORT_623 |
2871                                                E1000_MDEF_PORT_664));
2872                                 manc2h |= (1 << 1);
2873                                 j++;
2874                                 break;
2875                         }
2876
2877                 if (!j)
2878                         e_warn("Unable to create IPMI pass-through filter\n");
2879                 break;
2880         }
2881
2882         ew32(MANC2H, manc2h);
2883         ew32(MANC, manc);
2884 }
2885
2886 /**
2887  * e1000_configure_tx - Configure Transmit Unit after Reset
2888  * @adapter: board private structure
2889  *
2890  * Configure the Tx unit of the MAC after a reset.
2891  **/
2892 static void e1000_configure_tx(struct e1000_adapter *adapter)
2893 {
2894         struct e1000_hw *hw = &adapter->hw;
2895         struct e1000_ring *tx_ring = adapter->tx_ring;
2896         u64 tdba;
2897         u32 tdlen, tarc;
2898
2899         /* Setup the HW Tx Head and Tail descriptor pointers */
2900         tdba = tx_ring->dma;
2901         tdlen = tx_ring->count * sizeof(struct e1000_tx_desc);
2902         ew32(TDBAL(0), (tdba & DMA_BIT_MASK(32)));
2903         ew32(TDBAH(0), (tdba >> 32));