drivers/net/ethernet/qlogic/qlcnic/qlcnic_io.c

   1 /*
   2  * QLogic qlcnic NIC Driver
   3  * Copyright (c) 2009-2013 QLogic Corporation
   4  *
   5  * See LICENSE.qlcnic for copyright and licensing details.
   6  */
   7
   8 #include <linux/netdevice.h>
   9 #include <linux/if_vlan.h>
  10 #include <net/ip.h>
  11 #include <linux/ipv6.h>
  12 #include <net/checksum.h>
  13
  14 #include "qlcnic.h"
  15
  16 #define QLCNIC_TX_ETHER_PKT             0x01
  17 #define QLCNIC_TX_TCP_PKT               0x02
  18 #define QLCNIC_TX_UDP_PKT               0x03
  19 #define QLCNIC_TX_IP_PKT                0x04
  20 #define QLCNIC_TX_TCP_LSO               0x05
  21 #define QLCNIC_TX_TCP_LSO6              0x06
  22 #define QLCNIC_TX_ENCAP_PKT             0x07
  23 #define QLCNIC_TX_ENCAP_LSO             0x08
  24 #define QLCNIC_TX_TCPV6_PKT             0x0b
  25 #define QLCNIC_TX_UDPV6_PKT             0x0c
  26
  27 #define QLCNIC_FLAGS_VLAN_TAGGED        0x10
  28 #define QLCNIC_FLAGS_VLAN_OOB           0x40
  29
  30 #define qlcnic_set_tx_vlan_tci(cmd_desc, v)     \
  31         (cmd_desc)->vlan_TCI = cpu_to_le16(v);
  32 #define qlcnic_set_cmd_desc_port(cmd_desc, var) \
  33         ((cmd_desc)->port_ctxid |= ((var) & 0x0F))
  34 #define qlcnic_set_cmd_desc_ctxid(cmd_desc, var)        \
  35         ((cmd_desc)->port_ctxid |= ((var) << 4 & 0xF0))
  36
  37 #define qlcnic_set_tx_port(_desc, _port) \
  38         ((_desc)->port_ctxid = ((_port) & 0xf) | (((_port) << 4) & 0xf0))
  39
  40 #define qlcnic_set_tx_flags_opcode(_desc, _flags, _opcode) \
  41         ((_desc)->flags_opcode |= \
  42         cpu_to_le16(((_flags) & 0x7f) | (((_opcode) & 0x3f) << 7)))
  43
  44 #define qlcnic_set_tx_frags_len(_desc, _frags, _len) \
  45         ((_desc)->nfrags__length = \
  46         cpu_to_le32(((_frags) & 0xff) | (((_len) & 0xffffff) << 8)))
  47
  48 /* owner bits of status_desc */
  49 #define STATUS_OWNER_HOST       (0x1ULL << 56)
  50 #define STATUS_OWNER_PHANTOM    (0x2ULL << 56)
  51
  52 /* Status descriptor:
  53    0-3 port, 4-7 status, 8-11 type, 12-27 total_length
  54    28-43 reference_handle, 44-47 protocol, 48-52 pkt_offset
  55    53-55 desc_cnt, 56-57 owner, 58-63 opcode
  56  */
  57 #define qlcnic_get_sts_port(sts_data)   \
  58         ((sts_data) & 0x0F)
  59 #define qlcnic_get_sts_status(sts_data) \
  60         (((sts_data) >> 4) & 0x0F)
  61 #define qlcnic_get_sts_type(sts_data)   \
  62         (((sts_data) >> 8) & 0x0F)
  63 #define qlcnic_get_sts_totallength(sts_data)    \
  64         (((sts_data) >> 12) & 0xFFFF)
  65 #define qlcnic_get_sts_refhandle(sts_data)      \
  66         (((sts_data) >> 28) & 0xFFFF)
  67 #define qlcnic_get_sts_prot(sts_data)   \
  68         (((sts_data) >> 44) & 0x0F)
  69 #define qlcnic_get_sts_pkt_offset(sts_data)     \
  70         (((sts_data) >> 48) & 0x1F)
  71 #define qlcnic_get_sts_desc_cnt(sts_data)       \
  72         (((sts_data) >> 53) & 0x7)
  73 #define qlcnic_get_sts_opcode(sts_data) \
  74         (((sts_data) >> 58) & 0x03F)
  75
  76 #define qlcnic_get_lro_sts_refhandle(sts_data)  \
  77         ((sts_data) & 0x07FFF)
  78 #define qlcnic_get_lro_sts_length(sts_data)     \
  79         (((sts_data) >> 16) & 0x0FFFF)
  80 #define qlcnic_get_lro_sts_l2_hdr_offset(sts_data)      \
  81         (((sts_data) >> 32) & 0x0FF)
  82 #define qlcnic_get_lro_sts_l4_hdr_offset(sts_data)      \
  83         (((sts_data) >> 40) & 0x0FF)
  84 #define qlcnic_get_lro_sts_timestamp(sts_data)  \
  85         (((sts_data) >> 48) & 0x1)
  86 #define qlcnic_get_lro_sts_type(sts_data)       \
  87         (((sts_data) >> 49) & 0x7)
  88 #define qlcnic_get_lro_sts_push_flag(sts_data)          \
  89         (((sts_data) >> 52) & 0x1)
  90 #define qlcnic_get_lro_sts_seq_number(sts_data)         \
  91         ((sts_data) & 0x0FFFFFFFF)
  92 #define qlcnic_get_lro_sts_mss(sts_data1)               \
  93         ((sts_data1 >> 32) & 0x0FFFF)
  94
  95 #define qlcnic_83xx_get_lro_sts_mss(sts) ((sts) & 0xffff)
  96
  97 /* opcode field in status_desc */
  98 #define QLCNIC_SYN_OFFLOAD      0x03
  99 #define QLCNIC_RXPKT_DESC       0x04
 100 #define QLCNIC_OLD_RXPKT_DESC   0x3f
 101 #define QLCNIC_RESPONSE_DESC    0x05
 102 #define QLCNIC_LRO_DESC         0x12
 103
 104 #define QLCNIC_TX_POLL_BUDGET           128
 105 #define QLCNIC_TCP_HDR_SIZE             20
 106 #define QLCNIC_TCP_TS_OPTION_SIZE       12
 107 #define QLCNIC_FETCH_RING_ID(handle)    ((handle) >> 63)
 108 #define QLCNIC_DESC_OWNER_FW            cpu_to_le64(STATUS_OWNER_PHANTOM)
 109
 110 #define QLCNIC_TCP_TS_HDR_SIZE (QLCNIC_TCP_HDR_SIZE + QLCNIC_TCP_TS_OPTION_SIZE)
 111
 112 /* for status field in status_desc */
 113 #define STATUS_CKSUM_LOOP       0
 114 #define STATUS_CKSUM_OK         2
 115
 116 #define qlcnic_83xx_pktln(sts)          ((sts >> 32) & 0x3FFF)
 117 #define qlcnic_83xx_hndl(sts)           ((sts >> 48) & 0x7FFF)
 118 #define qlcnic_83xx_csum_status(sts)    ((sts >> 39) & 7)
 119 #define qlcnic_83xx_opcode(sts) ((sts >> 42) & 0xF)
 120 #define qlcnic_83xx_vlan_tag(sts)       (((sts) >> 48) & 0xFFFF)
 121 #define qlcnic_83xx_lro_pktln(sts)      (((sts) >> 32) & 0x3FFF)
 122 #define qlcnic_83xx_l2_hdr_off(sts)     (((sts) >> 16) & 0xFF)
 123 #define qlcnic_83xx_l4_hdr_off(sts)     (((sts) >> 24) & 0xFF)
 124 #define qlcnic_83xx_pkt_cnt(sts)        (((sts) >> 16) & 0x7)
 125 #define qlcnic_83xx_is_tstamp(sts)      (((sts) >> 40) & 1)
 126 #define qlcnic_83xx_is_psh_bit(sts)     (((sts) >> 41) & 1)
 127 #define qlcnic_83xx_is_ip_align(sts)    (((sts) >> 46) & 1)
 128 #define qlcnic_83xx_has_vlan_tag(sts)   (((sts) >> 47) & 1)
 129
 130 static int qlcnic_process_rcv_ring(struct qlcnic_host_sds_ring *sds_ring,
 131                                    int max);
 132
 133 static struct sk_buff *qlcnic_process_rxbuf(struct qlcnic_adapter *,
 134                                             struct qlcnic_host_rds_ring *,
 135                                             u16, u16);
 136
 137 static inline u8 qlcnic_mac_hash(u64 mac, u16 vlan)
 138 {
 139         return (u8)((mac & 0xff) ^ ((mac >> 40) & 0xff) ^ (vlan & 0xff));
 140 }
 141
 142 static inline u32 qlcnic_get_ref_handle(struct qlcnic_adapter *adapter,
 143                                         u16 handle, u8 ring_id)
 144 {
 145         if (qlcnic_83xx_check(adapter))
 146                 return handle | (ring_id << 15);
 147         else
 148                 return handle;
 149 }
 150
 151 static inline int qlcnic_82xx_is_lb_pkt(u64 sts_data)
 152 {
 153         return (qlcnic_get_sts_status(sts_data) == STATUS_CKSUM_LOOP) ? 1 : 0;
 154 }
 155
 156 static void qlcnic_delete_rx_list_mac(struct qlcnic_adapter *adapter,
 157                                       struct qlcnic_filter *fil,
 158                                       void *addr, u16 vlan_id)
 159 {
 160         int ret;
 161         u8 op;
 162
 163         op = vlan_id ? QLCNIC_MAC_VLAN_ADD : QLCNIC_MAC_ADD;
 164         ret = qlcnic_sre_macaddr_change(adapter, addr, vlan_id, op);
 165         if (ret)
 166                 return;
 167
 168         op = vlan_id ? QLCNIC_MAC_VLAN_DEL : QLCNIC_MAC_DEL;
 169         ret = qlcnic_sre_macaddr_change(adapter, addr, vlan_id, op);
 170         if (!ret) {
 171                 hlist_del(&fil->fnode);
 172                 adapter->rx_fhash.fnum--;
 173         }
 174 }
 175
 176 static struct qlcnic_filter *qlcnic_find_mac_filter(struct hlist_head *head,
 177                                                     void *addr, u16 vlan_id)
 178 {
 179         struct qlcnic_filter *tmp_fil = NULL;
 180         struct hlist_node *n;
 181
 182         hlist_for_each_entry_safe(tmp_fil, n, head, fnode) {
 183                 if (ether_addr_equal(tmp_fil->faddr, addr) &&
 184                     tmp_fil->vlan_id == vlan_id)
 185                         return tmp_fil;
 186         }
 187
 188         return NULL;
 189 }
 190
 191 static void qlcnic_add_lb_filter(struct qlcnic_adapter *adapter,
 192                                  struct sk_buff *skb, int loopback_pkt, u16 vlan_id)
 193 {
 194         struct ethhdr *phdr = (struct ethhdr *)(skb->data);
 195         struct qlcnic_filter *fil, *tmp_fil;
 196         struct hlist_head *head;
 197         unsigned long time;
 198         u64 src_addr = 0;
 199         u8 hindex, op;
 200         int ret;
 201
 202         if (!qlcnic_sriov_pf_check(adapter) || (vlan_id == 0xffff))
 203                 vlan_id = 0;
 204
 205         memcpy(&src_addr, phdr->h_source, ETH_ALEN);
 206         hindex = qlcnic_mac_hash(src_addr, vlan_id) &
 207                  (adapter->fhash.fbucket_size - 1);
 208
 209         if (loopback_pkt) {
 210                 if (adapter->rx_fhash.fnum >= adapter->rx_fhash.fmax)
 211                         return;
 212
 213                 head = &(adapter->rx_fhash.fhead[hindex]);
 214
 215                 tmp_fil = qlcnic_find_mac_filter(head, &src_addr, vlan_id);
 216                 if (tmp_fil) {
 217                         time = tmp_fil->ftime;
 218                         if (time_after(jiffies, QLCNIC_READD_AGE * HZ + time))
 219                                 tmp_fil->ftime = jiffies;
 220                         return;
 221                 }
 222
 223                 fil = kzalloc(sizeof(struct qlcnic_filter), GFP_ATOMIC);
 224                 if (!fil)
 225                         return;
 226
 227                 fil->ftime = jiffies;
 228                 memcpy(fil->faddr, &src_addr, ETH_ALEN);
 229                 fil->vlan_id = vlan_id;
 230                 spin_lock(&adapter->rx_mac_learn_lock);
 231                 hlist_add_head(&(fil->fnode), head);
 232                 adapter->rx_fhash.fnum++;
 233                 spin_unlock(&adapter->rx_mac_learn_lock);
 234         } else {
 235                 head = &adapter->fhash.fhead[hindex];
 236
 237                 spin_lock(&adapter->mac_learn_lock);
 238
 239                 tmp_fil = qlcnic_find_mac_filter(head, &src_addr, vlan_id);
 240                 if (tmp_fil) {
 241                         op = vlan_id ? QLCNIC_MAC_VLAN_DEL : QLCNIC_MAC_DEL;
 242                         ret = qlcnic_sre_macaddr_change(adapter,
 243                                                         (u8 *)&src_addr,
 244                                                         vlan_id, op);
 245                         if (!ret) {
 246                                 hlist_del(&tmp_fil->fnode);
 247                                 adapter->fhash.fnum--;
 248                         }
 249
 250                         spin_unlock(&adapter->mac_learn_lock);
 251
 252                         return;
 253                 }
 254
 255                 spin_unlock(&adapter->mac_learn_lock);
 256
 257                 head = &adapter->rx_fhash.fhead[hindex];
 258
 259                 spin_lock(&adapter->rx_mac_learn_lock);
 260
 261                 tmp_fil = qlcnic_find_mac_filter(head, &src_addr, vlan_id);
 262                 if (tmp_fil)
 263                         qlcnic_delete_rx_list_mac(adapter, tmp_fil, &src_addr,
 264                                                   vlan_id);
 265
 266                 spin_unlock(&adapter->rx_mac_learn_lock);
 267         }
 268 }
 269
 270 void qlcnic_82xx_change_filter(struct qlcnic_adapter *adapter, u64 *uaddr,
 271                                u16 vlan_id)
 272 {
 273         struct cmd_desc_type0 *hwdesc;
 274         struct qlcnic_nic_req *req;
 275         struct qlcnic_mac_req *mac_req;
 276         struct qlcnic_vlan_req *vlan_req;
 277         struct qlcnic_host_tx_ring *tx_ring = adapter->tx_ring;
 278         u32 producer;
 279         u64 word;
 280
 281         producer = tx_ring->producer;
 282         hwdesc = &tx_ring->desc_head[tx_ring->producer];
 283
 284         req = (struct qlcnic_nic_req *)hwdesc;
 285         memset(req, 0, sizeof(struct qlcnic_nic_req));
 286         req->qhdr = cpu_to_le64(QLCNIC_REQUEST << 23);
 287
 288         word = QLCNIC_MAC_EVENT | ((u64)(adapter->portnum) << 16);
 289         req->req_hdr = cpu_to_le64(word);
 290
 291         mac_req = (struct qlcnic_mac_req *)&(req->words[0]);
 292         mac_req->op = vlan_id ? QLCNIC_MAC_VLAN_ADD : QLCNIC_MAC_ADD;
 293         memcpy(mac_req->mac_addr, uaddr, ETH_ALEN);
 294
 295         vlan_req = (struct qlcnic_vlan_req *)&req->words[1];
 296         vlan_req->vlan_id = cpu_to_le16(vlan_id);
 297
 298         tx_ring->producer = get_next_index(producer, tx_ring->num_desc);
 299         smp_mb();
 300 }
 301
 302 static void qlcnic_send_filter(struct qlcnic_adapter *adapter,
 303                                struct cmd_desc_type0 *first_desc,
 304                                struct sk_buff *skb)
 305 {
 306         struct vlan_ethhdr *vh = (struct vlan_ethhdr *)(skb->data);
 307         struct ethhdr *phdr = (struct ethhdr *)(skb->data);
 308         u16 protocol = ntohs(skb->protocol);
 309         struct qlcnic_filter *fil, *tmp_fil;
 310         struct hlist_head *head;
 311         struct hlist_node *n;
 312         u64 src_addr = 0;
 313         u16 vlan_id = 0;
 314         u8 hindex, hval;
 315
 316         if (ether_addr_equal(phdr->h_source, adapter->mac_addr))
 317                 return;
 318
 319         if (adapter->flags & QLCNIC_VLAN_FILTERING) {
 320                 if (protocol == ETH_P_8021Q) {
 321                         vh = (struct vlan_ethhdr *)skb->data;
 322                         vlan_id = ntohs(vh->h_vlan_TCI);
 323                 } else if (vlan_tx_tag_present(skb)) {
 324                         vlan_id = vlan_tx_tag_get(skb);
 325                 }
 326         }
 327
 328         memcpy(&src_addr, phdr->h_source, ETH_ALEN);
 329         hval = qlcnic_mac_hash(src_addr, vlan_id);
 330         hindex = hval & (adapter->fhash.fbucket_size - 1);
 331         head = &(adapter->fhash.fhead[hindex]);
 332
 333         hlist_for_each_entry_safe(tmp_fil, n, head, fnode) {
 334                 if (ether_addr_equal(tmp_fil->faddr, (u8 *)&src_addr) &&
 335                     tmp_fil->vlan_id == vlan_id) {
 336                         if (jiffies > (QLCNIC_READD_AGE * HZ + tmp_fil->ftime))
 337                                 qlcnic_change_filter(adapter, &src_addr,
 338                                                      vlan_id);
 339                         tmp_fil->ftime = jiffies;
 340                         return;
 341                 }
 342         }
 343
 344         if (unlikely(adapter->fhash.fnum >= adapter->fhash.fmax)) {
 345                 adapter->stats.mac_filter_limit_overrun++;
 346                 return;
 347         }
 348
 349         fil = kzalloc(sizeof(struct qlcnic_filter), GFP_ATOMIC);
 350         if (!fil)
 351                 return;
 352
 353         qlcnic_change_filter(adapter, &src_addr, vlan_id);
 354         fil->ftime = jiffies;
 355         fil->vlan_id = vlan_id;
 356         memcpy(fil->faddr, &src_addr, ETH_ALEN);
 357         spin_lock(&adapter->mac_learn_lock);
 358         hlist_add_head(&(fil->fnode), head);
 359         adapter->fhash.fnum++;
 360         spin_unlock(&adapter->mac_learn_lock);
 361 }
 362
 363 #define QLCNIC_ENCAP_VXLAN_PKT          BIT_0
 364 #define QLCNIC_ENCAP_OUTER_L3_IP6       BIT_1
 365 #define QLCNIC_ENCAP_INNER_L3_IP6       BIT_2
 366 #define QLCNIC_ENCAP_INNER_L4_UDP       BIT_3
 367 #define QLCNIC_ENCAP_DO_L3_CSUM         BIT_4
 368 #define QLCNIC_ENCAP_DO_L4_CSUM         BIT_5
 369
 370 static int qlcnic_tx_encap_pkt(struct qlcnic_adapter *adapter,
 371                                struct cmd_desc_type0 *first_desc,
 372                                struct sk_buff *skb,
 373                                struct qlcnic_host_tx_ring *tx_ring)
 374 {
 375         u8 opcode = 0, inner_hdr_len = 0, outer_hdr_len = 0, total_hdr_len = 0;
 376         int copied, copy_len, descr_size;
 377         u32 producer = tx_ring->producer;
 378         struct cmd_desc_type0 *hwdesc;
 379         u16 flags = 0, encap_descr = 0;
 380
 381         opcode = QLCNIC_TX_ETHER_PKT;
 382         encap_descr = QLCNIC_ENCAP_VXLAN_PKT;
 383
 384         if (skb_is_gso(skb)) {
 385                 inner_hdr_len = skb_inner_transport_header(skb) +
 386                                 inner_tcp_hdrlen(skb) -
 387                                 skb_inner_mac_header(skb);
 388
 389                 /* VXLAN header size = 8 */
 390                 outer_hdr_len = skb_transport_offset(skb) + 8 +
 391                                 sizeof(struct udphdr);
 392                 first_desc->outer_hdr_length = outer_hdr_len;
 393                 total_hdr_len = inner_hdr_len + outer_hdr_len;
 394                 encap_descr |= QLCNIC_ENCAP_DO_L3_CSUM |
 395                                QLCNIC_ENCAP_DO_L4_CSUM;
 396                 first_desc->mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
 397                 first_desc->hdr_length = inner_hdr_len;
 398
 399                 /* Copy inner and outer headers in Tx descriptor(s)
 400                  * If total_hdr_len > cmd_desc_type0, use multiple
 401                  * descriptors
 402                  */
 403                 copied = 0;
 404                 descr_size = (int)sizeof(struct cmd_desc_type0);
 405                 while (copied < total_hdr_len) {
 406                         copy_len = min(descr_size, (total_hdr_len - copied));
 407                         hwdesc = &tx_ring->desc_head[producer];
 408                         tx_ring->cmd_buf_arr[producer].skb = NULL;
 409                         skb_copy_from_linear_data_offset(skb, copied,
 410                                                          (char *)hwdesc,
 411                                                          copy_len);
 412                         copied += copy_len;
 413                         producer = get_next_index(producer, tx_ring->num_desc);
 414                 }
 415
 416                 tx_ring->producer = producer;
 417
 418                 /* Make sure updated tx_ring->producer is visible
 419                  * for qlcnic_tx_avail()
 420                  */
 421                 smp_mb();
 422                 adapter->stats.encap_lso_frames++;
 423
 424                 opcode = QLCNIC_TX_ENCAP_LSO;
 425         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 426                 if (inner_ip_hdr(skb)->version == 6) {
 427                         if (inner_ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)
 428                                 encap_descr |= QLCNIC_ENCAP_INNER_L4_UDP;
 429                 } else {
 430                         if (inner_ip_hdr(skb)->protocol == IPPROTO_UDP)
 431                                 encap_descr |= QLCNIC_ENCAP_INNER_L4_UDP;
 432                 }
 433
 434                 adapter->stats.encap_tx_csummed++;
 435                 opcode = QLCNIC_TX_ENCAP_PKT;
 436         }
 437
 438         /* Prepare first 16 bits of byte offset 16 of Tx descriptor */
 439         if (ip_hdr(skb)->version == 6)
 440                 encap_descr |= QLCNIC_ENCAP_OUTER_L3_IP6;
 441
 442         /* outer IP header's size in 32bit words size*/
 443         encap_descr |= (skb_network_header_len(skb) >> 2) << 6;
 444
 445         /* outer IP header offset */
 446         encap_descr |= skb_network_offset(skb) << 10;
 447         first_desc->encap_descr = cpu_to_le16(encap_descr);
 448
 449         first_desc->tcp_hdr_offset = skb_inner_transport_header(skb) -
 450                                      skb->data;
 451         first_desc->ip_hdr_offset = skb_inner_network_offset(skb);
 452
 453         qlcnic_set_tx_flags_opcode(first_desc, flags, opcode);
 454
 455         return 0;
 456 }
 457
 458 static int qlcnic_tx_pkt(struct qlcnic_adapter *adapter,
 459                          struct cmd_desc_type0 *first_desc, struct sk_buff *skb,
 460                          struct qlcnic_host_tx_ring *tx_ring)
 461 {
 462         u8 l4proto, opcode = 0, hdr_len = 0;
 463         u16 flags = 0, vlan_tci = 0;
 464         int copied, offset, copy_len, size;
 465         struct cmd_desc_type0 *hwdesc;
 466         struct vlan_ethhdr *vh;
 467         u16 protocol = ntohs(skb->protocol);
 468         u32 producer = tx_ring->producer;
 469
 470         if (protocol == ETH_P_8021Q) {
 471                 vh = (struct vlan_ethhdr *)skb->data;
 472                 flags = QLCNIC_FLAGS_VLAN_TAGGED;
 473                 vlan_tci = ntohs(vh->h_vlan_TCI);
 474                 protocol = ntohs(vh->h_vlan_encapsulated_proto);
 475         } else if (vlan_tx_tag_present(skb)) {
 476                 flags = QLCNIC_FLAGS_VLAN_OOB;
 477                 vlan_tci = vlan_tx_tag_get(skb);
 478         }
 479         if (unlikely(adapter->tx_pvid)) {
 480                 if (vlan_tci && !(adapter->flags & QLCNIC_TAGGING_ENABLED))
 481                         return -EIO;
 482                 if (vlan_tci && (adapter->flags & QLCNIC_TAGGING_ENABLED))
 483                         goto set_flags;
 484
 485                 flags = QLCNIC_FLAGS_VLAN_OOB;
 486                 vlan_tci = adapter->tx_pvid;
 487         }
 488 set_flags:
 489         qlcnic_set_tx_vlan_tci(first_desc, vlan_tci);
 490         qlcnic_set_tx_flags_opcode(first_desc, flags, opcode);
 491
 492         if (*(skb->data) & BIT_0) {
 493                 flags |= BIT_0;
 494                 memcpy(&first_desc->eth_addr, skb->data, ETH_ALEN);
 495         }
 496         opcode = QLCNIC_TX_ETHER_PKT;
 497         if (skb_is_gso(skb)) {
 498                 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
 499                 first_desc->mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
 500                 first_desc->hdr_length = hdr_len;
 501                 opcode = (protocol == ETH_P_IPV6) ? QLCNIC_TX_TCP_LSO6 :
 502                                                     QLCNIC_TX_TCP_LSO;
 503
 504                 /* For LSO, we need to copy the MAC/IP/TCP headers into
 505                 * the descriptor ring */
 506                 copied = 0;
 507                 offset = 2;
 508
 509                 if (flags & QLCNIC_FLAGS_VLAN_OOB) {
 510                         first_desc->hdr_length += VLAN_HLEN;
 511                         first_desc->tcp_hdr_offset = VLAN_HLEN;
 512                         first_desc->ip_hdr_offset = VLAN_HLEN;
 513
 514                         /* Only in case of TSO on vlan device */
 515                         flags |= QLCNIC_FLAGS_VLAN_TAGGED;
 516
 517                         /* Create a TSO vlan header template for firmware */
 518                         hwdesc = &tx_ring->desc_head[producer];
 519                         tx_ring->cmd_buf_arr[producer].skb = NULL;
 520
 521                         copy_len = min((int)sizeof(struct cmd_desc_type0) -
 522                                        offset, hdr_len + VLAN_HLEN);
 523
 524                         vh = (struct vlan_ethhdr *)((char *) hwdesc + 2);
 525                         skb_copy_from_linear_data(skb, vh, 12);
 526                         vh->h_vlan_proto = htons(ETH_P_8021Q);
 527                         vh->h_vlan_TCI = htons(vlan_tci);
 528
 529                         skb_copy_from_linear_data_offset(skb, 12,
 530                                                          (char *)vh + 16,
 531                                                          copy_len - 16);
 532                         copied = copy_len - VLAN_HLEN;
 533                         offset = 0;
 534                         producer = get_next_index(producer, tx_ring->num_desc);
 535                 }
 536
 537                 while (copied < hdr_len) {
 538                         size = (int)sizeof(struct cmd_desc_type0) - offset;
 539                         copy_len = min(size, (hdr_len - copied));
 540                         hwdesc = &tx_ring->desc_head[producer];
 541                         tx_ring->cmd_buf_arr[producer].skb = NULL;
 542                         skb_copy_from_linear_data_offset(skb, copied,
 543                                                          (char *)hwdesc +
 544                                                          offset, copy_len);
 545                         copied += copy_len;
 546                         offset = 0;
 547                         producer = get_next_index(producer, tx_ring->num_desc);
 548                 }
 549
 550                 tx_ring->producer = producer;
 551                 smp_mb();
 552                 adapter->stats.lso_frames++;
 553
 554         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 555                 if (protocol == ETH_P_IP) {
 556                         l4proto = ip_hdr(skb)->protocol;
 557
 558                         if (l4proto == IPPROTO_TCP)
 559                                 opcode = QLCNIC_TX_TCP_PKT;
 560                         else if (l4proto == IPPROTO_UDP)
 561                                 opcode = QLCNIC_TX_UDP_PKT;
 562                 } else if (protocol == ETH_P_IPV6) {
 563                         l4proto = ipv6_hdr(skb)->nexthdr;
 564
 565                         if (l4proto == IPPROTO_TCP)
 566                                 opcode = QLCNIC_TX_TCPV6_PKT;
 567                         else if (l4proto == IPPROTO_UDP)
 568                                 opcode = QLCNIC_TX_UDPV6_PKT;
 569                 }
 570         }
 571         first_desc->tcp_hdr_offset += skb_transport_offset(skb);
 572         first_desc->ip_hdr_offset += skb_network_offset(skb);
 573         qlcnic_set_tx_flags_opcode(first_desc, flags, opcode);
 574
 575         return 0;
 576 }
 577
 578 static int qlcnic_map_tx_skb(struct pci_dev *pdev, struct sk_buff *skb,
 579                              struct qlcnic_cmd_buffer *pbuf)
 580 {
 581         struct qlcnic_skb_frag *nf;
 582         struct skb_frag_struct *frag;
 583         int i, nr_frags;
 584         dma_addr_t map;
 585
 586         nr_frags = skb_shinfo(skb)->nr_frags;
 587         nf = &pbuf->frag_array[0];
 588
 589         map = pci_map_single(pdev, skb->data, skb_headlen(skb),
 590                              PCI_DMA_TODEVICE);
 591         if (pci_dma_mapping_error(pdev, map))
 592                 goto out_err;
 593
 594         nf->dma = map;
 595         nf->length = skb_headlen(skb);
 596
 597         for (i = 0; i < nr_frags; i++) {
 598                 frag = &skb_shinfo(skb)->frags[i];
 599                 nf = &pbuf->frag_array[i+1];
 600                 map = skb_frag_dma_map(&pdev->dev, frag, 0, skb_frag_size(frag),
 601                                        DMA_TO_DEVICE);
 602                 if (dma_mapping_error(&pdev->dev, map))
 603                         goto unwind;
 604
 605                 nf->dma = map;
 606                 nf->length = skb_frag_size(frag);
 607         }
 608
 609         return 0;
 610
 611 unwind:
 612         while (--i >= 0) {
 613                 nf = &pbuf->frag_array[i+1];
 614                 pci_unmap_page(pdev, nf->dma, nf->length, PCI_DMA_TODEVICE);
 615         }
 616
 617         nf = &pbuf->frag_array[0];
 618         pci_unmap_single(pdev, nf->dma, skb_headlen(skb), PCI_DMA_TODEVICE);
 619
 620 out_err:
 621         return -ENOMEM;
 622 }
 623
 624 static void qlcnic_unmap_buffers(struct pci_dev *pdev, struct sk_buff *skb,
 625                                  struct qlcnic_cmd_buffer *pbuf)
 626 {
 627         struct qlcnic_skb_frag *nf = &pbuf->frag_array[0];
 628         int i, nr_frags = skb_shinfo(skb)->nr_frags;
 629
 630         for (i = 0; i < nr_frags; i++) {
 631                 nf = &pbuf->frag_array[i+1];
 632                 pci_unmap_page(pdev, nf->dma, nf->length, PCI_DMA_TODEVICE);
 633         }
 634
 635         nf = &pbuf->frag_array[0];
 636         pci_unmap_single(pdev, nf->dma, skb_headlen(skb), PCI_DMA_TODEVICE);
 637         pbuf->skb = NULL;
 638 }
 639
 640 static inline void qlcnic_clear_cmddesc(u64 *desc)
 641 {
 642         desc[0] = 0ULL;
 643         desc[2] = 0ULL;
 644         desc[7] = 0ULL;
 645 }
 646
 647 netdev_tx_t qlcnic_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
 648 {
 649         struct qlcnic_adapter *adapter = netdev_priv(netdev);
 650         struct qlcnic_host_tx_ring *tx_ring;
 651         struct qlcnic_cmd_buffer *pbuf;
 652         struct qlcnic_skb_frag *buffrag;
 653         struct cmd_desc_type0 *hwdesc, *first_desc;
 654         struct pci_dev *pdev;
 655         struct ethhdr *phdr;
 656         int i, k, frag_count, delta = 0;
 657         u32 producer, num_txd;
 658         u16 protocol;
 659         bool l4_is_udp = false;
 660
 661         if (!test_bit(__QLCNIC_DEV_UP, &adapter->state)) {
 662                 netif_tx_stop_all_queues(netdev);
 663                 return NETDEV_TX_BUSY;
 664         }
 665
 666         if (adapter->flags & QLCNIC_MACSPOOF) {
 667                 phdr = (struct ethhdr *)skb->data;
 668                 if (!ether_addr_equal(phdr->h_source, adapter->mac_addr))
 669                         goto drop_packet;
 670         }
 671
 672         tx_ring = &adapter->tx_ring[skb_get_queue_mapping(skb)];
 673         num_txd = tx_ring->num_desc;
 674
 675         frag_count = skb_shinfo(skb)->nr_frags + 1;
 676
 677         /* 14 frags supported for normal packet and
 678          * 32 frags supported for TSO packet
 679          */
 680         if (!skb_is_gso(skb) && frag_count > QLCNIC_MAX_FRAGS_PER_TX) {
 681                 for (i = 0; i < (frag_count - QLCNIC_MAX_FRAGS_PER_TX); i++)
 682                         delta += skb_frag_size(&skb_shinfo(skb)->frags[i]);
 683
 684                 if (!__pskb_pull_tail(skb, delta))
 685                         goto drop_packet;
 686
 687                 frag_count = 1 + skb_shinfo(skb)->nr_frags;
 688         }
 689
 690         if (unlikely(qlcnic_tx_avail(tx_ring) <= TX_STOP_THRESH)) {
 691                 netif_tx_stop_queue(tx_ring->txq);
 692                 if (qlcnic_tx_avail(tx_ring) > TX_STOP_THRESH) {
 693                         netif_tx_start_queue(tx_ring->txq);
 694                 } else {
 695                         tx_ring->tx_stats.xmit_off++;
 696                         return NETDEV_TX_BUSY;
 697                 }
 698         }
 699
 700         producer = tx_ring->producer;
 701         pbuf = &tx_ring->cmd_buf_arr[producer];
 702         pdev = adapter->pdev;
 703         first_desc = &tx_ring->desc_head[producer];
 704         hwdesc = &tx_ring->desc_head[producer];
 705         qlcnic_clear_cmddesc((u64 *)hwdesc);
 706
 707         if (qlcnic_map_tx_skb(pdev, skb, pbuf)) {
 708                 adapter->stats.tx_dma_map_error++;
 709                 goto drop_packet;
 710         }
 711
 712         pbuf->skb = skb;
 713         pbuf->frag_count = frag_count;
 714
 715         qlcnic_set_tx_frags_len(first_desc, frag_count, skb->len);
 716         qlcnic_set_tx_port(first_desc, adapter->portnum);
 717
 718         for (i = 0; i < frag_count; i++) {
 719                 k = i % 4;
 720
 721                 if ((k == 0) && (i > 0)) {
 722                         /* move to next desc.*/
 723                         producer = get_next_index(producer, num_txd);
 724                         hwdesc = &tx_ring->desc_head[producer];
 725                         qlcnic_clear_cmddesc((u64 *)hwdesc);
 726                         tx_ring->cmd_buf_arr[producer].skb = NULL;
 727                 }
 728
 729                 buffrag = &pbuf->frag_array[i];
 730                 hwdesc->buffer_length[k] = cpu_to_le16(buffrag->length);
 731                 switch (k) {
 732                 case 0:
 733                         hwdesc->addr_buffer1 = cpu_to_le64(buffrag->dma);
 734                         break;
 735                 case 1:
 736                         hwdesc->addr_buffer2 = cpu_to_le64(buffrag->dma);
 737                         break;
 738                 case 2:
 739                         hwdesc->addr_buffer3 = cpu_to_le64(buffrag->dma);
 740                         break;
 741                 case 3:
 742                         hwdesc->addr_buffer4 = cpu_to_le64(buffrag->dma);
 743                         break;
 744                 }
 745         }
 746
 747         tx_ring->producer = get_next_index(producer, num_txd);
 748         smp_mb();
 749
 750         protocol = ntohs(skb->protocol);
 751         if (protocol == ETH_P_IP)
 752                 l4_is_udp = ip_hdr(skb)->protocol == IPPROTO_UDP;
 753         else if (protocol == ETH_P_IPV6)
 754                 l4_is_udp = ipv6_hdr(skb)->nexthdr == IPPROTO_UDP;
 755
 756         /* Check if it is a VXLAN packet */
 757         if (!skb->encapsulation || !l4_is_udp ||
 758             !qlcnic_encap_tx_offload(adapter)) {
 759                 if (unlikely(qlcnic_tx_pkt(adapter, first_desc, skb,
 760                                            tx_ring)))
 761                         goto unwind_buff;
 762         } else {
 763                 if (unlikely(qlcnic_tx_encap_pkt(adapter, first_desc,
 764                                                  skb, tx_ring)))
 765                         goto unwind_buff;
 766         }
 767
 768         if (adapter->drv_mac_learn)
 769                 qlcnic_send_filter(adapter, first_desc, skb);
 770
 771         tx_ring->tx_stats.tx_bytes += skb->len;
 772         tx_ring->tx_stats.xmit_called++;
 773
 774         qlcnic_update_cmd_producer(tx_ring);
 775
 776         return NETDEV_TX_OK;
 777
 778 unwind_buff:
 779         qlcnic_unmap_buffers(pdev, skb, pbuf);
 780 drop_packet:
 781         adapter->stats.txdropped++;
 782         dev_kfree_skb_any(skb);
 783         return NETDEV_TX_OK;
 784 }
 785
 786 void qlcnic_advert_link_change(struct qlcnic_adapter *adapter, int linkup)
 787 {
 788         struct net_device *netdev = adapter->netdev;
 789
 790         if (adapter->ahw->linkup && !linkup) {
 791                 netdev_info(netdev, "NIC Link is down\n");
 792                 adapter->ahw->linkup = 0;
 793                 netif_carrier_off(netdev);
 794         } else if (!adapter->ahw->linkup && linkup) {
 795                 adapter->ahw->linkup = 1;
 796
 797                 /* Do not advertise Link up to the stack if device
 798                  * is in loopback mode
 799                  */
 800                 if (qlcnic_83xx_check(adapter) && adapter->ahw->lb_mode) {
 801                         netdev_info(netdev, "NIC Link is up for loopback test\n");
 802                         return;
 803                 }
 804
 805                 netdev_info(netdev, "NIC Link is up\n");
 806                 netif_carrier_on(netdev);
 807         }
 808 }
 809
 810 static int qlcnic_alloc_rx_skb(struct qlcnic_adapter *adapter,
 811                                struct qlcnic_host_rds_ring *rds_ring,
 812                                struct qlcnic_rx_buffer *buffer)
 813 {
 814         struct sk_buff *skb;
 815         dma_addr_t dma;
 816         struct pci_dev *pdev = adapter->pdev;
 817
 818         skb = netdev_alloc_skb(adapter->netdev, rds_ring->skb_size);
 819         if (!skb) {
 820                 adapter->stats.skb_alloc_failure++;
 821                 return -ENOMEM;
 822         }
 823
 824         skb_reserve(skb, NET_IP_ALIGN);
 825         dma = pci_map_single(pdev, skb->data,
 826                              rds_ring->dma_size, PCI_DMA_FROMDEVICE);
 827
 828         if (pci_dma_mapping_error(pdev, dma)) {
 829                 adapter->stats.rx_dma_map_error++;
 830                 dev_kfree_skb_any(skb);
 831                 return -ENOMEM;
 832         }
 833
 834         buffer->skb = skb;
 835         buffer->dma = dma;
 836
 837         return 0;
 838 }
 839
 840 static void qlcnic_post_rx_buffers_nodb(struct qlcnic_adapter *adapter,
 841                                         struct qlcnic_host_rds_ring *rds_ring,
 842                                         u8 ring_id)
 843 {
 844         struct rcv_desc *pdesc;
 845         struct qlcnic_rx_buffer *buffer;
 846         int  count = 0;
 847         uint32_t producer, handle;
 848         struct list_head *head;
 849
 850         if (!spin_trylock(&rds_ring->lock))
 851                 return;
 852
 853         producer = rds_ring->producer;
 854         head = &rds_ring->free_list;
 855         while (!list_empty(head)) {
 856                 buffer = list_entry(head->next, struct qlcnic_rx_buffer, list);
 857
 858                 if (!buffer->skb) {
 859                         if (qlcnic_alloc_rx_skb(adapter, rds_ring, buffer))
 860                                 break;
 861                 }
 862                 count++;
 863                 list_del(&buffer->list);
 864
 865                 /* make a rcv descriptor  */
 866                 pdesc = &rds_ring->desc_head[producer];
 867                 handle = qlcnic_get_ref_handle(adapter,
 868                                                buffer->ref_handle, ring_id);
 869                 pdesc->reference_handle = cpu_to_le16(handle);
 870                 pdesc->buffer_length = cpu_to_le32(rds_ring->dma_size);
 871                 pdesc->addr_buffer = cpu_to_le64(buffer->dma);
 872                 producer = get_next_index(producer, rds_ring->num_desc);
 873         }
 874         if (count) {
 875                 rds_ring->producer = producer;
 876                 writel((producer - 1) & (rds_ring->num_desc - 1),
 877                        rds_ring->crb_rcv_producer);
 878         }
 879         spin_unlock(&rds_ring->lock);
 880 }
 881
 882 static int qlcnic_process_cmd_ring(struct qlcnic_adapter *adapter,
 883                                    struct qlcnic_host_tx_ring *tx_ring,
 884                                    int budget)
 885 {
 886         u32 sw_consumer, hw_consumer;
 887         int i, done, count = 0;
 888         struct qlcnic_cmd_buffer *buffer;
 889         struct pci_dev *pdev = adapter->pdev;
 890         struct net_device *netdev = adapter->netdev;
 891         struct qlcnic_skb_frag *frag;
 892
 893         if (!spin_trylock(&tx_ring->tx_clean_lock))
 894                 return 1;
 895
 896         sw_consumer = tx_ring->sw_consumer;
 897         hw_consumer = le32_to_cpu(*(tx_ring->hw_consumer));
 898
 899         while (sw_consumer != hw_consumer) {
 900                 buffer = &tx_ring->cmd_buf_arr[sw_consumer];
 901                 if (buffer->skb) {
 902                         frag = &buffer->frag_array[0];
 903                         pci_unmap_single(pdev, frag->dma, frag->length,
 904                                          PCI_DMA_TODEVICE);
 905                         frag->dma = 0ULL;
 906                         for (i = 1; i < buffer->frag_count; i++) {
 907                                 frag++;
 908                                 pci_unmap_page(pdev, frag->dma, frag->length,
 909                                                PCI_DMA_TODEVICE);
 910                                 frag->dma = 0ULL;
 911                         }
 912                         tx_ring->tx_stats.xmit_finished++;
 913                         dev_kfree_skb_any(buffer->skb);
 914                         buffer->skb = NULL;
 915                 }
 916
 917                 sw_consumer = get_next_index(sw_consumer, tx_ring->num_desc);
 918                 if (++count >= budget)
 919                         break;
 920         }
 921
 922         tx_ring->sw_consumer = sw_consumer;
 923
 924         if (count && netif_running(netdev)) {
 925                 smp_mb();
 926                 if (netif_tx_queue_stopped(tx_ring->txq) &&
 927                     netif_carrier_ok(netdev)) {
 928                         if (qlcnic_tx_avail(tx_ring) > TX_STOP_THRESH) {
 929                                 netif_tx_wake_queue(tx_ring->txq);
 930                                 tx_ring->tx_stats.xmit_on++;
 931                         }
 932                 }
 933                 adapter->tx_timeo_cnt = 0;
 934         }
 935         /*
 936          * If everything is freed up to consumer then check if the ring is full
 937          * If the ring is full then check if more needs to be freed and
 938          * schedule the call back again.
 939          *
 940          * This happens when there are 2 CPUs. One could be freeing and the
 941          * other filling it. If the ring is full when we get out of here and
 942          * the card has already interrupted the host then the host can miss the
 943          * interrupt.
 944          *
 945          * There is still a possible race condition and the host could miss an
 946          * interrupt. The card has to take care of this.
 947          */
 948         hw_consumer = le32_to_cpu(*(tx_ring->hw_consumer));
 949         done = (sw_consumer == hw_consumer);
 950
 951         spin_unlock(&tx_ring->tx_clean_lock);
 952
 953         return done;
 954 }
 955
 956 static int qlcnic_poll(struct napi_struct *napi, int budget)
 957 {
 958         int tx_complete, work_done;
 959         struct qlcnic_host_sds_ring *sds_ring;
 960         struct qlcnic_adapter *adapter;
 961         struct qlcnic_host_tx_ring *tx_ring;
 962
 963         sds_ring = container_of(napi, struct qlcnic_host_sds_ring, napi);
 964         adapter = sds_ring->adapter;
 965         tx_ring = sds_ring->tx_ring;
 966
 967         tx_complete = qlcnic_process_cmd_ring(adapter, tx_ring,
 968                                               budget);
 969         work_done = qlcnic_process_rcv_ring(sds_ring, budget);
 970
 971         /* Check if we need a repoll */
 972         if (!tx_complete)
 973                 work_done = budget;
 974
 975         if (work_done < budget) {
 976                 napi_complete(&sds_ring->napi);
 977                 if (test_bit(__QLCNIC_DEV_UP, &adapter->state)) {
 978                         qlcnic_enable_sds_intr(adapter, sds_ring);
 979                         qlcnic_enable_tx_intr(adapter, tx_ring);
 980                 }
 981         }
 982
 983         return work_done;
 984 }
 985
 986 static int qlcnic_tx_poll(struct napi_struct *napi, int budget)
 987 {
 988         struct qlcnic_host_tx_ring *tx_ring;
 989         struct qlcnic_adapter *adapter;
 990         int work_done;
 991
 992         tx_ring = container_of(napi, struct qlcnic_host_tx_ring, napi);
 993         adapter = tx_ring->adapter;
 994
 995         work_done = qlcnic_process_cmd_ring(adapter, tx_ring, budget);
 996         if (work_done) {
 997                 napi_complete(&tx_ring->napi);
 998                 if (test_bit(__QLCNIC_DEV_UP, &adapter->state))
 999                         qlcnic_enable_tx_intr(adapter, tx_ring);
1000         } else {
1001                 /* As qlcnic_process_cmd_ring() returned 0, we need a repoll*/
1002                 work_done = budget;
1003         }
1004
1005         return work_done;
1006 }
1007
1008 static int qlcnic_rx_poll(struct napi_struct *napi, int budget)
1009 {
1010         struct qlcnic_host_sds_ring *sds_ring;
1011         struct qlcnic_adapter *adapter;
1012         int work_done;
1013
1014         sds_ring = container_of(napi, struct qlcnic_host_sds_ring, napi);
1015         adapter = sds_ring->adapter;
1016
1017         work_done = qlcnic_process_rcv_ring(sds_ring, budget);
1018
1019         if (work_done < budget) {
1020                 napi_complete(&sds_ring->napi);
1021                 if (test_bit(__QLCNIC_DEV_UP, &adapter->state))
1022                         qlcnic_enable_sds_intr(adapter, sds_ring);
1023         }
1024
1025         return work_done;
1026 }
1027
1028 static void qlcnic_handle_linkevent(struct qlcnic_adapter *adapter,
1029                                     struct qlcnic_fw_msg *msg)
1030 {
1031         u32 cable_OUI;
1032         u16 cable_len, link_speed;
1033         u8  link_status, module, duplex, autoneg, lb_status = 0;
1034         struct net_device *netdev = adapter->netdev;
1035
1036         adapter->ahw->has_link_events = 1;
1037
1038         cable_OUI = msg->body[1] & 0xffffffff;
1039         cable_len = (msg->body[1] >> 32) & 0xffff;
1040         link_speed = (msg->body[1] >> 48) & 0xffff;
1041
1042         link_status = msg->body[2] & 0xff;
1043         duplex = (msg->body[2] >> 16) & 0xff;
1044         autoneg = (msg->body[2] >> 24) & 0xff;
1045         lb_status = (msg->body[2] >> 32) & 0x3;
1046
1047         module = (msg->body[2] >> 8) & 0xff;
1048         if (module == LINKEVENT_MODULE_TWINAX_UNSUPPORTED_CABLE)
1049                 dev_info(&netdev->dev,
1050                          "unsupported cable: OUI 0x%x, length %d\n",
1051                          cable_OUI, cable_len);
1052         else if (module == LINKEVENT_MODULE_TWINAX_UNSUPPORTED_CABLELEN)
1053                 dev_info(&netdev->dev, "unsupported cable length %d\n",
1054                          cable_len);
1055
1056         if (!link_status && (lb_status == QLCNIC_ILB_MODE ||
1057             lb_status == QLCNIC_ELB_MODE))
1058                 adapter->ahw->loopback_state |= QLCNIC_LINKEVENT;
1059
1060         qlcnic_advert_link_change(adapter, link_status);
1061
1062         if (duplex == LINKEVENT_FULL_DUPLEX)
1063                 adapter->ahw->link_duplex = DUPLEX_FULL;
1064         else
1065                 adapter->ahw->link_duplex = DUPLEX_HALF;
1066
1067         adapter->ahw->module_type = module;
1068         adapter->ahw->link_autoneg = autoneg;
1069
1070         if (link_status) {
1071                 adapter->ahw->link_speed = link_speed;
1072         } else {
1073                 adapter->ahw->link_speed = SPEED_UNKNOWN;
1074                 adapter->ahw->link_duplex = DUPLEX_UNKNOWN;
1075         }
1076 }
1077
1078 static void qlcnic_handle_fw_message(int desc_cnt, int index,
1079                                      struct qlcnic_host_sds_ring *sds_ring)
1080 {
1081         struct qlcnic_fw_msg msg;
1082         struct status_desc *desc;
1083         struct qlcnic_adapter *adapter;
1084         struct device *dev;
1085         int i = 0, opcode, ret;
1086
1087         while (desc_cnt > 0 && i < 8) {
1088                 desc = &sds_ring->desc_head[index];
1089                 msg.words[i++] = le64_to_cpu(desc->status_desc_data[0]);
1090                 msg.words[i++] = le64_to_cpu(desc->status_desc_data[1]);
1091
1092                 index = get_next_index(index, sds_ring->num_desc);
1093                 desc_cnt--;
1094         }
1095
1096         adapter = sds_ring->adapter;
1097         dev = &adapter->pdev->dev;
1098         opcode = qlcnic_get_nic_msg_opcode(msg.body[0]);
1099
1100         switch (opcode) {
1101         case QLCNIC_C2H_OPCODE_GET_LINKEVENT_RESPONSE:
1102                 qlcnic_handle_linkevent(adapter, &msg);
1103                 break;
1104         case QLCNIC_C2H_OPCODE_CONFIG_LOOPBACK:
1105                 ret = (u32)(msg.body[1]);
1106                 switch (ret) {
1107                 case 0:
1108                         adapter->ahw->loopback_state |= QLCNIC_LB_RESPONSE;
1109                         break;
1110                 case 1:
1111                         dev_info(dev, "loopback already in progress\n");
1112                         adapter->ahw->diag_cnt = -EINPROGRESS;
1113                         break;
1114                 case 2:
1115                         dev_info(dev, "loopback cable is not connected\n");
1116                         adapter->ahw->diag_cnt = -ENODEV;
1117                         break;
1118                 default:
1119                         dev_info(dev,
1120                                  "loopback configure request failed, err %x\n",
1121                                  ret);
1122                         adapter->ahw->diag_cnt = -EIO;
1123                         break;
1124                 }
1125                 break;
1126         case QLCNIC_C2H_OPCODE_GET_DCB_AEN:
1127                 qlcnic_dcb_aen_handler(adapter->dcb, (void *)&msg);
1128                 break;
1129         default:
1130                 break;
1131         }
1132 }
1133
1134 static struct sk_buff *qlcnic_process_rxbuf(struct qlcnic_adapter *adapter,
1135                                             struct qlcnic_host_rds_ring *ring,
1136                                             u16 index, u16 cksum)
1137 {
1138         struct qlcnic_rx_buffer *buffer;
1139         struct sk_buff *skb;
1140
1141         buffer = &ring->rx_buf_arr[index];
1142         if (unlikely(buffer->skb == NULL)) {
1143                 WARN_ON(1);
1144                 return NULL;
1145         }
1146
1147         pci_unmap_single(adapter->pdev, buffer->dma, ring->dma_size,
1148                          PCI_DMA_FROMDEVICE);
1149
1150         skb = buffer->skb;
1151         if (likely((adapter->netdev->features & NETIF_F_RXCSUM) &&
1152                    (cksum == STATUS_CKSUM_OK || cksum == STATUS_CKSUM_LOOP))) {
1153                 adapter->stats.csummed++;
1154                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1155         } else {
1156                 skb_checksum_none_assert(skb);
1157         }
1158
1159
1160         buffer->skb = NULL;
1161
1162         return skb;
1163 }
1164
1165 static inline int qlcnic_check_rx_tagging(struct qlcnic_adapter *adapter,
1166                                           struct sk_buff *skb, u16 *vlan_tag)
1167 {
1168         struct ethhdr *eth_hdr;
1169
1170         if (!__vlan_get_tag(skb, vlan_tag)) {
1171                 eth_hdr = (struct ethhdr *)skb->data;
1172                 memmove(skb->data + VLAN_HLEN, eth_hdr, ETH_ALEN * 2);
1173                 skb_pull(skb, VLAN_HLEN);
1174         }
1175         if (!adapter->rx_pvid)
1176                 return 0;
1177
1178         if (*vlan_tag == adapter->rx_pvid) {
1179                 /* Outer vlan tag. Packet should follow non-vlan path */
1180                 *vlan_tag = 0xffff;
1181                 return 0;
1182         }
1183         if (adapter->flags & QLCNIC_TAGGING_ENABLED)
1184                 return 0;
1185
1186         return -EINVAL;
1187 }
1188
1189 static struct qlcnic_rx_buffer *
1190 qlcnic_process_rcv(struct qlcnic_adapter *adapter,
1191                    struct qlcnic_host_sds_ring *sds_ring, int ring,
1192                    u64 sts_data0)
1193 {
1194         struct net_device *netdev = adapter->netdev;
1195         struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
1196         struct qlcnic_rx_buffer *buffer;
1197         struct sk_buff *skb;
1198         struct qlcnic_host_rds_ring *rds_ring;
1199         int index, length, cksum, pkt_offset, is_lb_pkt;
1200         u16 vid = 0xffff, t_vid;
1201
1202         if (unlikely(ring >= adapter->max_rds_rings))
1203                 return NULL;
1204
1205         rds_ring = &recv_ctx->rds_rings[ring];
1206
1207         index = qlcnic_get_sts_refhandle(sts_data0);
1208         if (unlikely(index >= rds_ring->num_desc))
1209                 return NULL;
1210
1211         buffer = &rds_ring->rx_buf_arr[index];
1212         length = qlcnic_get_sts_totallength(sts_data0);
1213         cksum  = qlcnic_get_sts_status(sts_data0);
1214         pkt_offset = qlcnic_get_sts_pkt_offset(sts_data0);
1215
1216         skb = qlcnic_process_rxbuf(adapter, rds_ring, index, cksum);
1217         if (!skb)
1218                 return buffer;
1219
1220         if (adapter->rx_mac_learn) {
1221                 t_vid = 0;
1222                 is_lb_pkt = qlcnic_82xx_is_lb_pkt(sts_data0);
1223                 qlcnic_add_lb_filter(adapter, skb, is_lb_pkt, t_vid);
1224         }
1225
1226         if (length > rds_ring->skb_size)
1227                 skb_put(skb, rds_ring->skb_size);
1228         else
1229                 skb_put(skb, length);
1230
1231         if (pkt_offset)
1232                 skb_pull(skb, pkt_offset);
1233
1234         if (unlikely(qlcnic_check_rx_tagging(adapter, skb, &vid))) {
1235                 adapter->stats.rxdropped++;
1236                 dev_kfree_skb(skb);
1237                 return buffer;
1238         }
1239
1240         skb->protocol = eth_type_trans(skb, netdev);
1241
1242         if (vid != 0xffff)
1243                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
1244
1245         napi_gro_receive(&sds_ring->napi, skb);
1246
1247         adapter->stats.rx_pkts++;
1248         adapter->stats.rxbytes += length;
1249
1250         return buffer;
1251 }
1252
1253 #define QLC_TCP_HDR_SIZE            20
1254 #define QLC_TCP_TS_OPTION_SIZE      12
1255 #define QLC_TCP_TS_HDR_SIZE         (QLC_TCP_HDR_SIZE + QLC_TCP_TS_OPTION_SIZE)
1256
1257 static struct qlcnic_rx_buffer *
1258 qlcnic_process_lro(struct qlcnic_adapter *adapter,
1259                    int ring, u64 sts_data0, u64 sts_data1)
1260 {
1261         struct net_device *netdev = adapter->netdev;
1262         struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
1263         struct qlcnic_rx_buffer *buffer;
1264         struct sk_buff *skb;
1265         struct qlcnic_host_rds_ring *rds_ring;
1266         struct iphdr *iph;
1267         struct ipv6hdr *ipv6h;
1268         struct tcphdr *th;
1269         bool push, timestamp;
1270         int index, l2_hdr_offset, l4_hdr_offset, is_lb_pkt;
1271         u16 lro_length, length, data_offset, t_vid, vid = 0xffff;
1272         u32 seq_number;
1273
1274         if (unlikely(ring >= adapter->max_rds_rings))
1275                 return NULL;
1276
1277         rds_ring = &recv_ctx->rds_rings[ring];
1278
1279         index = qlcnic_get_lro_sts_refhandle(sts_data0);
1280         if (unlikely(index >= rds_ring->num_desc))
1281                 return NULL;
1282
1283         buffer = &rds_ring->rx_buf_arr[index];
1284
1285         timestamp = qlcnic_get_lro_sts_timestamp(sts_data0);
1286         lro_length = qlcnic_get_lro_sts_length(sts_data0);
1287         l2_hdr_offset = qlcnic_get_lro_sts_l2_hdr_offset(sts_data0);
1288         l4_hdr_offset = qlcnic_get_lro_sts_l4_hdr_offset(sts_data0);
1289         push = qlcnic_get_lro_sts_push_flag(sts_data0);
1290         seq_number = qlcnic_get_lro_sts_seq_number(sts_data1);
1291
1292         skb = qlcnic_process_rxbuf(adapter, rds_ring, index, STATUS_CKSUM_OK);
1293         if (!skb)
1294                 return buffer;
1295
1296         if (adapter->rx_mac_learn) {
1297                 t_vid = 0;
1298                 is_lb_pkt = qlcnic_82xx_is_lb_pkt(sts_data0);
1299                 qlcnic_add_lb_filter(adapter, skb, is_lb_pkt, t_vid);
1300         }
1301
1302         if (timestamp)
1303                 data_offset = l4_hdr_offset + QLC_TCP_TS_HDR_SIZE;
1304         else
1305                 data_offset = l4_hdr_offset + QLC_TCP_HDR_SIZE;
1306
1307         skb_put(skb, lro_length + data_offset);
1308         skb_pull(skb, l2_hdr_offset);
1309
1310         if (unlikely(qlcnic_check_rx_tagging(adapter, skb, &vid))) {
1311                 adapter->stats.rxdropped++;
1312                 dev_kfree_skb(skb);
1313                 return buffer;
1314         }
1315
1316         skb->protocol = eth_type_trans(skb, netdev);
1317
1318         if (ntohs(skb->protocol) == ETH_P_IPV6) {
1319                 ipv6h = (struct ipv6hdr *)skb->data;
1320                 th = (struct tcphdr *)(skb->data + sizeof(struct ipv6hdr));
1321                 length = (th->doff << 2) + lro_length;
1322                 ipv6h->payload_len = htons(length);
1323         } else {
1324                 iph = (struct iphdr *)skb->data;
1325                 th = (struct tcphdr *)(skb->data + (iph->ihl << 2));
1326                 length = (iph->ihl << 2) + (th->doff << 2) + lro_length;
1327                 csum_replace2(&iph->check, iph->tot_len, htons(length));
1328                 iph->tot_len = htons(length);
1329         }
1330
1331         th->psh = push;
1332         th->seq = htonl(seq_number);
1333         length = skb->len;
1334
1335         if (adapter->flags & QLCNIC_FW_LRO_MSS_CAP) {
1336                 skb_shinfo(skb)->gso_size = qlcnic_get_lro_sts_mss(sts_data1);
1337                 if (skb->protocol == htons(ETH_P_IPV6))
1338                         skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
1339                 else
1340                         skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
1341         }
1342
1343         if (vid != 0xffff)
1344                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
1345         netif_receive_skb(skb);
1346
1347         adapter->stats.lro_pkts++;
1348         adapter->stats.lrobytes += length;
1349
1350         return buffer;
1351 }
1352
1353 static int qlcnic_process_rcv_ring(struct qlcnic_host_sds_ring *sds_ring, int max)
1354 {
1355         struct qlcnic_host_rds_ring *rds_ring;
1356         struct qlcnic_adapter *adapter = sds_ring->adapter;
1357         struct list_head *cur;
1358         struct status_desc *desc;
1359         struct qlcnic_rx_buffer *rxbuf;
1360         int opcode, desc_cnt, count = 0;
1361         u64 sts_data0, sts_data1;
1362         u8 ring;
1363         u32 consumer = sds_ring->consumer;
1364
1365         while (count < max) {
1366                 desc = &sds_ring->desc_head[consumer];
1367                 sts_data0 = le64_to_cpu(desc->status_desc_data[0]);
1368
1369                 if (!(sts_data0 & STATUS_OWNER_HOST))
1370                         break;
1371
1372                 desc_cnt = qlcnic_get_sts_desc_cnt(sts_data0);
1373                 opcode = qlcnic_get_sts_opcode(sts_data0);
1374                 switch (opcode) {
1375                 case QLCNIC_RXPKT_DESC:
1376                 case QLCNIC_OLD_RXPKT_DESC:
1377                 case QLCNIC_SYN_OFFLOAD:
1378                         ring = qlcnic_get_sts_type(sts_data0);
1379                         rxbuf = qlcnic_process_rcv(adapter, sds_ring, ring,
1380                                                    sts_data0);
1381                         break;
1382                 case QLCNIC_LRO_DESC:
1383                         ring = qlcnic_get_lro_sts_type(sts_data0);
1384                         sts_data1 = le64_to_cpu(desc->status_desc_data[1]);
1385                         rxbuf = qlcnic_process_lro(adapter, ring, sts_data0,
1386                                                    sts_data1);
1387                         break;
1388                 case QLCNIC_RESPONSE_DESC:
1389                         qlcnic_handle_fw_message(desc_cnt, consumer, sds_ring);
1390                 default:
1391                         goto skip;
1392                 }
1393                 WARN_ON(desc_cnt > 1);
1394
1395                 if (likely(rxbuf))
1396                         list_add_tail(&rxbuf->list, &sds_ring->free_list[ring]);
1397                 else
1398                         adapter->stats.null_rxbuf++;
1399 skip:
1400                 for (; desc_cnt > 0; desc_cnt--) {
1401                         desc = &sds_ring->desc_head[consumer];
1402                         desc->status_desc_data[0] = QLCNIC_DESC_OWNER_FW;
1403                         consumer = get_next_index(consumer, sds_ring->num_desc);
1404                 }
1405                 count++;
1406         }
1407
1408         for (ring = 0; ring < adapter->max_rds_rings; ring++) {
1409                 rds_ring = &adapter->recv_ctx->rds_rings[ring];
1410                 if (!list_empty(&sds_ring->free_list[ring])) {
1411                         list_for_each(cur, &sds_ring->free_list[ring]) {
1412                                 rxbuf = list_entry(cur, struct qlcnic_rx_buffer,
1413                                                    list);
1414                                 qlcnic_alloc_rx_skb(adapter, rds_ring, rxbuf);
1415                         }
1416                         spin_lock(&rds_ring->lock);
1417                         list_splice_tail_init(&sds_ring->free_list[ring],
1418                                               &rds_ring->free_list);
1419                         spin_unlock(&rds_ring->lock);
1420                 }
1421
1422                 qlcnic_post_rx_buffers_nodb(adapter, rds_ring, ring);
1423         }
1424
1425         if (count) {
1426                 sds_ring->consumer = consumer;
1427                 writel(consumer, sds_ring->crb_sts_consumer);
1428         }
1429
1430         return count;
1431 }
1432
1433 void qlcnic_post_rx_buffers(struct qlcnic_adapter *adapter,
1434                             struct qlcnic_host_rds_ring *rds_ring, u8 ring_id)
1435 {
1436         struct rcv_desc *pdesc;
1437         struct qlcnic_rx_buffer *buffer;
1438         int count = 0;
1439         u32 producer, handle;
1440         struct list_head *head;
1441
1442         producer = rds_ring->producer;
1443         head = &rds_ring->free_list;
1444
1445         while (!list_empty(head)) {
1446
1447                 buffer = list_entry(head->next, struct qlcnic_rx_buffer, list);
1448
1449                 if (!buffer->skb) {
1450                         if (qlcnic_alloc_rx_skb(adapter, rds_ring, buffer))
1451                                 break;
1452                 }
1453
1454                 count++;
1455                 list_del(&buffer->list);
1456
1457                 /* make a rcv descriptor  */
1458                 pdesc = &rds_ring->desc_head[producer];
1459                 pdesc->addr_buffer = cpu_to_le64(buffer->dma);
1460                 handle = qlcnic_get_ref_handle(adapter, buffer->ref_handle,
1461                                                ring_id);
1462                 pdesc->reference_handle = cpu_to_le16(handle);
1463                 pdesc->buffer_length = cpu_to_le32(rds_ring->dma_size);
1464                 producer = get_next_index(producer, rds_ring->num_desc);
1465         }
1466
1467         if (count) {
1468                 rds_ring->producer = producer;
1469                 writel((producer-1) & (rds_ring->num_desc-1),
1470                        rds_ring->crb_rcv_producer);
1471         }
1472 }
1473
1474 static void dump_skb(struct sk_buff *skb, struct qlcnic_adapter *adapter)
1475 {
1476         int i;
1477         unsigned char *data = skb->data;
1478
1479         pr_info(KERN_INFO "\n");
1480         for (i = 0; i < skb->len; i++) {
1481                 QLCDB(adapter, DRV, "%02x ", data[i]);
1482                 if ((i & 0x0f) == 8)
1483                         pr_info(KERN_INFO "\n");
1484         }
1485 }
1486
1487 static void qlcnic_process_rcv_diag(struct qlcnic_adapter *adapter, int ring,
1488                                     u64 sts_data0)
1489 {
1490         struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
1491         struct sk_buff *skb;
1492         struct qlcnic_host_rds_ring *rds_ring;
1493         int index, length, cksum, pkt_offset;
1494
1495         if (unlikely(ring >= adapter->max_rds_rings))
1496                 return;
1497
1498         rds_ring = &recv_ctx->rds_rings[ring];
1499
1500         index = qlcnic_get_sts_refhandle(sts_data0);
1501         length = qlcnic_get_sts_totallength(sts_data0);
1502         if (unlikely(index >= rds_ring->num_desc))
1503                 return;
1504
1505         cksum  = qlcnic_get_sts_status(sts_data0);
1506         pkt_offset = qlcnic_get_sts_pkt_offset(sts_data0);
1507
1508         skb = qlcnic_process_rxbuf(adapter, rds_ring, index, cksum);
1509         if (!skb)
1510                 return;
1511
1512         if (length > rds_ring->skb_size)
1513                 skb_put(skb, rds_ring->skb_size);
1514         else
1515                 skb_put(skb, length);
1516
1517         if (pkt_offset)
1518                 skb_pull(skb, pkt_offset);
1519
1520         if (!qlcnic_check_loopback_buff(skb->data, adapter->mac_addr))
1521                 adapter->ahw->diag_cnt++;
1522         else
1523                 dump_skb(skb, adapter);
1524
1525         dev_kfree_skb_any(skb);
1526         adapter->stats.rx_pkts++;
1527         adapter->stats.rxbytes += length;
1528
1529         return;
1530 }
1531
1532 void qlcnic_82xx_process_rcv_ring_diag(struct qlcnic_host_sds_ring *sds_ring)
1533 {
1534         struct qlcnic_adapter *adapter = sds_ring->adapter;
1535         struct status_desc *desc;
1536         u64 sts_data0;
1537         int ring, opcode, desc_cnt;
1538
1539         u32 consumer = sds_ring->consumer;
1540
1541         desc = &sds_ring->desc_head[consumer];
1542         sts_data0 = le64_to_cpu(desc->status_desc_data[0]);
1543
1544         if (!(sts_data0 & STATUS_OWNER_HOST))
1545                 return;
1546
1547         desc_cnt = qlcnic_get_sts_desc_cnt(sts_data0);
1548         opcode = qlcnic_get_sts_opcode(sts_data0);
1549         switch (opcode) {
1550         case QLCNIC_RESPONSE_DESC:
1551                 qlcnic_handle_fw_message(desc_cnt, consumer, sds_ring);
1552                 break;
1553         default:
1554                 ring = qlcnic_get_sts_type(sts_data0);
1555                 qlcnic_process_rcv_diag(adapter, ring, sts_data0);
1556                 break;
1557         }
1558
1559         for (; desc_cnt > 0; desc_cnt--) {
1560                 desc = &sds_ring->desc_head[consumer];
1561                 desc->status_desc_data[0] = cpu_to_le64(STATUS_OWNER_PHANTOM);
1562                 consumer = get_next_index(consumer, sds_ring->num_desc);
1563         }
1564
1565         sds_ring->consumer = consumer;
1566         writel(consumer, sds_ring->crb_sts_consumer);
1567 }
1568
1569 int qlcnic_82xx_napi_add(struct qlcnic_adapter *adapter,
1570                          struct net_device *netdev)
1571 {
1572         int ring;
1573         struct qlcnic_host_sds_ring *sds_ring;
1574         struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
1575         struct qlcnic_host_tx_ring *tx_ring;
1576
1577         if (qlcnic_alloc_sds_rings(recv_ctx, adapter->drv_sds_rings))
1578                 return -ENOMEM;
1579
1580         for (ring = 0; ring < adapter->drv_sds_rings; ring++) {
1581                 sds_ring = &recv_ctx->sds_rings[ring];
1582                 if (qlcnic_check_multi_tx(adapter) &&
1583                     !adapter->ahw->diag_test) {
1584                         netif_napi_add(netdev, &sds_ring->napi, qlcnic_rx_poll,
1585                                        NAPI_POLL_WEIGHT);
1586                 } else {
1587                         if (ring == (adapter->drv_sds_rings - 1))
1588                                 netif_napi_add(netdev, &sds_ring->napi,
1589                                                qlcnic_poll,
1590                                                NAPI_POLL_WEIGHT);
1591                         else
1592                                 netif_napi_add(netdev, &sds_ring->napi,
1593                                                qlcnic_rx_poll,
1594                                                NAPI_POLL_WEIGHT);
1595                 }
1596         }
1597
1598         if (qlcnic_alloc_tx_rings(adapter, netdev)) {
1599                 qlcnic_free_sds_rings(recv_ctx);
1600                 return -ENOMEM;
1601         }
1602
1603         if (qlcnic_check_multi_tx(adapter) && !adapter->ahw->diag_test) {
1604                 for (ring = 0; ring < adapter->drv_tx_rings; ring++) {
1605                         tx_ring = &adapter->tx_ring[ring];
1606                         netif_napi_add(netdev, &tx_ring->napi, qlcnic_tx_poll,
1607                                        NAPI_POLL_WEIGHT);
1608                 }
1609         }
1610
1611         return 0;
1612 }
1613
1614 void qlcnic_82xx_napi_del(struct qlcnic_adapter *adapter)
1615 {
1616         int ring;
1617         struct qlcnic_host_sds_ring *sds_ring;
1618         struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
1619         struct qlcnic_host_tx_ring *tx_ring;
1620
1621         for (ring = 0; ring < adapter->drv_sds_rings; ring++) {
1622                 sds_ring = &recv_ctx->sds_rings[ring];
1623                 netif_napi_del(&sds_ring->napi);
1624         }
1625
1626         qlcnic_free_sds_rings(adapter->recv_ctx);
1627
1628         if (qlcnic_check_multi_tx(adapter) && !adapter->ahw->diag_test) {
1629                 for (ring = 0; ring < adapter->drv_tx_rings; ring++) {
1630                         tx_ring = &adapter->tx_ring[ring];
1631                         netif_napi_del(&tx_ring->napi);
1632                 }
1633         }
1634
1635         qlcnic_free_tx_rings(adapter);
1636 }
1637
1638 void qlcnic_82xx_napi_enable(struct qlcnic_adapter *adapter)
1639 {
1640         int ring;
1641         struct qlcnic_host_sds_ring *sds_ring;
1642         struct qlcnic_host_tx_ring *tx_ring;
1643         struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
1644
1645         if (adapter->is_up != QLCNIC_ADAPTER_UP_MAGIC)
1646                 return;
1647
1648         for (ring = 0; ring < adapter->drv_sds_rings; ring++) {
1649                 sds_ring = &recv_ctx->sds_rings[ring];
1650                 napi_enable(&sds_ring->napi);
1651                 qlcnic_enable_sds_intr(adapter, sds_ring);
1652         }
1653
1654         if (qlcnic_check_multi_tx(adapter) &&
1655             (adapter->flags & QLCNIC_MSIX_ENABLED) &&
1656             !adapter->ahw->diag_test) {
1657                 for (ring = 0; ring < adapter->drv_tx_rings; ring++) {
1658                         tx_ring = &adapter->tx_ring[ring];
1659                         napi_enable(&tx_ring->napi);
1660                         qlcnic_enable_tx_intr(adapter, tx_ring);
1661                 }
1662         }
1663 }
1664
1665 void qlcnic_82xx_napi_disable(struct qlcnic_adapter *adapter)
1666 {
1667         int ring;
1668         struct qlcnic_host_sds_ring *sds_ring;
1669         struct qlcnic_host_tx_ring *tx_ring;
1670         struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
1671
1672         if (adapter->is_up != QLCNIC_ADAPTER_UP_MAGIC)
1673                 return;
1674
1675         for (ring = 0; ring < adapter->drv_sds_rings; ring++) {
1676                 sds_ring = &recv_ctx->sds_rings[ring];
1677                 qlcnic_disable_sds_intr(adapter, sds_ring);
1678                 napi_synchronize(&sds_ring->napi);
1679                 napi_disable(&sds_ring->napi);
1680         }
1681
1682         if ((adapter->flags & QLCNIC_MSIX_ENABLED) &&
1683             !adapter->ahw->diag_test &&
1684             qlcnic_check_multi_tx(adapter)) {
1685                 for (ring = 0; ring < adapter->drv_tx_rings; ring++) {
1686                         tx_ring = &adapter->tx_ring[ring];
1687                         qlcnic_disable_tx_intr(adapter, tx_ring);
1688                         napi_synchronize(&tx_ring->napi);
1689                         napi_disable(&tx_ring->napi);
1690                 }
1691         }
1692 }
1693
1694 #define QLC_83XX_NORMAL_LB_PKT  (1ULL << 36)
1695 #define QLC_83XX_LRO_LB_PKT     (1ULL << 46)
1696
1697 static inline int qlcnic_83xx_is_lb_pkt(u64 sts_data, int lro_pkt)
1698 {
1699         if (lro_pkt)
1700                 return (sts_data & QLC_83XX_LRO_LB_PKT) ? 1 : 0;
1701         else
1702                 return (sts_data & QLC_83XX_NORMAL_LB_PKT) ? 1 : 0;
1703 }
1704
1705 #define QLCNIC_ENCAP_LENGTH_MASK        0x7f
1706
1707 static inline u8 qlcnic_encap_length(u64 sts_data)
1708 {
1709         return sts_data & QLCNIC_ENCAP_LENGTH_MASK;
1710 }
1711
1712 static struct qlcnic_rx_buffer *
1713 qlcnic_83xx_process_rcv(struct qlcnic_adapter *adapter,
1714                         struct qlcnic_host_sds_ring *sds_ring,
1715                         u8 ring, u64 sts_data[])
1716 {
1717         struct net_device *netdev = adapter->netdev;
1718         struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
1719         struct qlcnic_rx_buffer *buffer;
1720         struct sk_buff *skb;
1721         struct qlcnic_host_rds_ring *rds_ring;
1722         int index, length, cksum, is_lb_pkt;
1723         u16 vid = 0xffff;
1724         int err;
1725
1726         if (unlikely(ring >= adapter->max_rds_rings))
1727                 return NULL;
1728
1729         rds_ring = &recv_ctx->rds_rings[ring];
1730
1731         index = qlcnic_83xx_hndl(sts_data[0]);
1732         if (unlikely(index >= rds_ring->num_desc))
1733                 return NULL;
1734
1735         buffer = &rds_ring->rx_buf_arr[index];
1736         length = qlcnic_83xx_pktln(sts_data[0]);
1737         cksum  = qlcnic_83xx_csum_status(sts_data[1]);
1738         skb = qlcnic_process_rxbuf(adapter, rds_ring, index, cksum);
1739         if (!skb)
1740                 return buffer;
1741
1742         if (length > rds_ring->skb_size)
1743                 skb_put(skb, rds_ring->skb_size);
1744         else
1745                 skb_put(skb, length);
1746
1747         err = qlcnic_check_rx_tagging(adapter, skb, &vid);
1748
1749         if (adapter->rx_mac_learn) {
1750                 is_lb_pkt = qlcnic_83xx_is_lb_pkt(sts_data[1], 0);
1751                 qlcnic_add_lb_filter(adapter, skb, is_lb_pkt, vid);
1752         }
1753
1754         if (unlikely(err)) {
1755                 adapter->stats.rxdropped++;
1756                 dev_kfree_skb(skb);
1757                 return buffer;
1758         }
1759
1760         skb->protocol = eth_type_trans(skb, netdev);
1761
1762         if (qlcnic_encap_length(sts_data[1]) &&
1763             skb->ip_summed == CHECKSUM_UNNECESSARY) {
1764                 skb->csum_level = 1;
1765                 adapter->stats.encap_rx_csummed++;
1766         }
1767
1768         if (vid != 0xffff)
1769                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
1770
1771         napi_gro_receive(&sds_ring->napi, skb);
1772
1773         adapter->stats.rx_pkts++;
1774         adapter->stats.rxbytes += length;
1775
1776         return buffer;
1777 }
1778
1779 static struct qlcnic_rx_buffer *
1780 qlcnic_83xx_process_lro(struct qlcnic_adapter *adapter,
1781                         u8 ring, u64 sts_data[])
1782 {
1783         struct net_device *netdev = adapter->netdev;
1784         struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
1785         struct qlcnic_rx_buffer *buffer;
1786         struct sk_buff *skb;
1787         struct qlcnic_host_rds_ring *rds_ring;
1788         struct iphdr *iph;
1789         struct ipv6hdr *ipv6h;
1790         struct tcphdr *th;
1791         bool push;
1792         int l2_hdr_offset, l4_hdr_offset;
1793         int index, is_lb_pkt;
1794         u16 lro_length, length, data_offset, gso_size;
1795         u16 vid = 0xffff;
1796         int err;
1797
1798         if (unlikely(ring >= adapter->max_rds_rings))
1799                 return NULL;
1800
1801         rds_ring = &recv_ctx->rds_rings[ring];
1802
1803         index = qlcnic_83xx_hndl(sts_data[0]);
1804         if (unlikely(index >= rds_ring->num_desc))
1805                 return NULL;
1806
1807         buffer = &rds_ring->rx_buf_arr[index];
1808
1809         lro_length = qlcnic_83xx_lro_pktln(sts_data[0]);
1810         l2_hdr_offset = qlcnic_83xx_l2_hdr_off(sts_data[1]);
1811         l4_hdr_offset = qlcnic_83xx_l4_hdr_off(sts_data[1]);
1812         push = qlcnic_83xx_is_psh_bit(sts_data[1]);
1813
1814         skb = qlcnic_process_rxbuf(adapter, rds_ring, index, STATUS_CKSUM_OK);
1815         if (!skb)
1816                 return buffer;
1817
1818         if (qlcnic_83xx_is_tstamp(sts_data[1]))
1819                 data_offset = l4_hdr_offset + QLCNIC_TCP_TS_HDR_SIZE;
1820         else
1821                 data_offset = l4_hdr_offset + QLCNIC_TCP_HDR_SIZE;
1822
1823         skb_put(skb, lro_length + data_offset);
1824         skb_pull(skb, l2_hdr_offset);
1825
1826         err = qlcnic_check_rx_tagging(adapter, skb, &vid);
1827
1828         if (adapter->rx_mac_learn) {
1829                 is_lb_pkt = qlcnic_83xx_is_lb_pkt(sts_data[1], 1);
1830                 qlcnic_add_lb_filter(adapter, skb, is_lb_pkt, vid);
1831         }
1832
1833         if (unlikely(err)) {
1834                 adapter->stats.rxdropped++;
1835                 dev_kfree_skb(skb);
1836                 return buffer;
1837         }
1838
1839         skb->protocol = eth_type_trans(skb, netdev);
1840         if (ntohs(skb->protocol) == ETH_P_IPV6) {
1841                 ipv6h = (struct ipv6hdr *)skb->data;
1842                 th = (struct tcphdr *)(skb->data + sizeof(struct ipv6hdr));
1843
1844                 length = (th->doff << 2) + lro_length;
1845                 ipv6h->payload_len = htons(length);
1846         } else {
1847                 iph = (struct iphdr *)skb->data;
1848                 th = (struct tcphdr *)(skb->data + (iph->ihl << 2));
1849                 length = (iph->ihl << 2) + (th->doff << 2) + lro_length;
1850                 csum_replace2(&iph->check, iph->tot_len, htons(length));
1851                 iph->tot_len = htons(length);
1852         }
1853
1854         th->psh = push;
1855         length = skb->len;
1856
1857         if (adapter->flags & QLCNIC_FW_LRO_MSS_CAP) {
1858                 gso_size = qlcnic_83xx_get_lro_sts_mss(sts_data[0]);
1859                 skb_shinfo(skb)->gso_size = gso_size;
1860                 if (skb->protocol == htons(ETH_P_IPV6))
1861                         skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
1862                 else
1863                         skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
1864         }
1865
1866         if (vid != 0xffff)
1867                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
1868
1869         netif_receive_skb(skb);
1870
1871         adapter->stats.lro_pkts++;
1872         adapter->stats.lrobytes += length;
1873         return buffer;
1874 }
1875
1876 static int qlcnic_83xx_process_rcv_ring(struct qlcnic_host_sds_ring *sds_ring,
1877                                         int max)
1878 {
1879         struct qlcnic_host_rds_ring *rds_ring;
1880         struct qlcnic_adapter *adapter = sds_ring->adapter;
1881         struct list_head *cur;
1882         struct status_desc *desc;
1883         struct qlcnic_rx_buffer *rxbuf = NULL;
1884         u8 ring;
1885         u64 sts_data[2];
1886         int count = 0, opcode;
1887         u32 consumer = sds_ring->consumer;
1888
1889         while (count < max) {
1890                 desc = &sds_ring->desc_head[consumer];
1891                 sts_data[1] = le64_to_cpu(desc->status_desc_data[1]);
1892                 opcode = qlcnic_83xx_opcode(sts_data[1]);
1893                 if (!opcode)
1894                         break;
1895                 sts_data[0] = le64_to_cpu(desc->status_desc_data[0]);
1896                 ring = QLCNIC_FETCH_RING_ID(sts_data[0]);
1897
1898                 switch (opcode) {
1899                 case QLC_83XX_REG_DESC:
1900                         rxbuf = qlcnic_83xx_process_rcv(adapter, sds_ring,
1901                                                         ring, sts_data);
1902                         break;
1903                 case QLC_83XX_LRO_DESC:
1904                         rxbuf = qlcnic_83xx_process_lro(adapter, ring,
1905                                                         sts_data);
1906                         break;
1907                 default:
1908                         dev_info(&adapter->pdev->dev,
1909                                  "Unknown opcode: 0x%x\n", opcode);
1910                         goto skip;
1911                 }
1912
1913                 if (likely(rxbuf))
1914                         list_add_tail(&rxbuf->list, &sds_ring->free_list[ring]);
1915                 else
1916                         adapter->stats.null_rxbuf++;
1917 skip:
1918                 desc = &sds_ring->desc_head[consumer];
1919                 /* Reset the descriptor */
1920                 desc->status_desc_data[1] = 0;
1921                 consumer = get_next_index(consumer, sds_ring->num_desc);
1922                 count++;
1923         }
1924         for (ring = 0; ring < adapter->max_rds_rings; ring++) {
1925                 rds_ring = &adapter->recv_ctx->rds_rings[ring];
1926                 if (!list_empty(&sds_ring->free_list[ring])) {
1927                         list_for_each(cur, &sds_ring->free_list[ring]) {
1928                                 rxbuf = list_entry(cur, struct qlcnic_rx_buffer,
1929                                                    list);
1930                                 qlcnic_alloc_rx_skb(adapter, rds_ring, rxbuf);
1931                         }
1932                         spin_lock(&rds_ring->lock);
1933                         list_splice_tail_init(&sds_ring->free_list[ring],
1934                                               &rds_ring->free_list);
1935                         spin_unlock(&rds_ring->lock);
1936                 }
1937                 qlcnic_post_rx_buffers_nodb(adapter, rds_ring, ring);
1938         }
1939         if (count) {
1940                 sds_ring->consumer = consumer;
1941                 writel(consumer, sds_ring->crb_sts_consumer);
1942         }
1943         return count;
1944 }
1945
1946 static int qlcnic_83xx_msix_sriov_vf_poll(struct napi_struct *napi, int budget)
1947 {
1948         int tx_complete;
1949         int work_done;
1950         struct qlcnic_host_sds_ring *sds_ring;
1951         struct qlcnic_adapter *adapter;
1952         struct qlcnic_host_tx_ring *tx_ring;
1953
1954         sds_ring = container_of(napi, struct qlcnic_host_sds_ring, napi);
1955         adapter = sds_ring->adapter;
1956         /* tx ring count = 1 */
1957         tx_ring = adapter->tx_ring;
1958
1959         tx_complete = qlcnic_process_cmd_ring(adapter, tx_ring, budget);
1960         work_done = qlcnic_83xx_process_rcv_ring(sds_ring, budget);
1961
1962         /* Check if we need a repoll */
1963         if (!tx_complete)
1964                 work_done = budget;
1965
1966         if (work_done < budget) {
1967                 napi_complete(&sds_ring->napi);
1968                 qlcnic_enable_sds_intr(adapter, sds_ring);
1969         }
1970
1971         return work_done;
1972 }
1973
1974 static int qlcnic_83xx_poll(struct napi_struct *napi, int budget)
1975 {
1976         int tx_complete;
1977         int work_done;
1978         struct qlcnic_host_sds_ring *sds_ring;
1979         struct qlcnic_adapter *adapter;
1980         struct qlcnic_host_tx_ring *tx_ring;
1981
1982         sds_ring = container_of(napi, struct qlcnic_host_sds_ring, napi);
1983         adapter = sds_ring->adapter;
1984         /* tx ring count = 1 */
1985         tx_ring = adapter->tx_ring;
1986
1987         tx_complete = qlcnic_process_cmd_ring(adapter, tx_ring, budget);
1988         work_done = qlcnic_83xx_process_rcv_ring(sds_ring, budget);
1989
1990         /* Check if we need a repoll */
1991         if (!tx_complete)
1992                 work_done = budget;
1993
1994         if (work_done < budget) {
1995                 napi_complete(&sds_ring->napi);
1996                 qlcnic_enable_sds_intr(adapter, sds_ring);
1997         }
1998
1999         return work_done;
2000 }
2001
2002 static int qlcnic_83xx_msix_tx_poll(struct napi_struct *napi, int budget)
2003 {
2004         int work_done;
2005         struct qlcnic_host_tx_ring *tx_ring;
2006         struct qlcnic_adapter *adapter;
2007
2008         budget = QLCNIC_TX_POLL_BUDGET;
2009         tx_ring = container_of(napi, struct qlcnic_host_tx_ring, napi);
2010         adapter = tx_ring->adapter;
2011         work_done = qlcnic_process_cmd_ring(adapter, tx_ring, budget);
2012         if (work_done) {
2013                 napi_complete(&tx_ring->napi);
2014                 if (test_bit(__QLCNIC_DEV_UP , &adapter->state))
2015                         qlcnic_enable_tx_intr(adapter, tx_ring);
2016         } else {
2017                 /* need a repoll */
2018                 work_done = budget;
2019         }
2020
2021         return work_done;
2022 }
2023
2024 static int qlcnic_83xx_rx_poll(struct napi_struct *napi, int budget)
2025 {
2026         int work_done;
2027         struct qlcnic_host_sds_ring *sds_ring;
2028         struct qlcnic_adapter *adapter;
2029
2030         sds_ring = container_of(napi, struct qlcnic_host_sds_ring, napi);
2031         adapter = sds_ring->adapter;
2032         work_done = qlcnic_83xx_process_rcv_ring(sds_ring, budget);
2033         if (work_done < budget) {
2034                 napi_complete(&sds_ring->napi);
2035                 if (test_bit(__QLCNIC_DEV_UP, &adapter->state))
2036                         qlcnic_enable_sds_intr(adapter, sds_ring);
2037         }
2038
2039         return work_done;
2040 }
2041
2042 void qlcnic_83xx_napi_enable(struct qlcnic_adapter *adapter)
2043 {
2044         int ring;
2045         struct qlcnic_host_sds_ring *sds_ring;
2046         struct qlcnic_host_tx_ring *tx_ring;
2047         struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
2048
2049         if (adapter->is_up != QLCNIC_ADAPTER_UP_MAGIC)
2050                 return;
2051
2052         for (ring = 0; ring < adapter->drv_sds_rings; ring++) {
2053                 sds_ring = &recv_ctx->sds_rings[ring];
2054                 napi_enable(&sds_ring->napi);
2055                 if (adapter->flags & QLCNIC_MSIX_ENABLED)
2056                         qlcnic_enable_sds_intr(adapter, sds_ring);
2057         }
2058
2059         if ((adapter->flags & QLCNIC_MSIX_ENABLED) &&
2060             !(adapter->flags & QLCNIC_TX_INTR_SHARED)) {
2061                 for (ring = 0; ring < adapter->drv_tx_rings; ring++) {
2062                         tx_ring = &adapter->tx_ring[ring];
2063                         napi_enable(&tx_ring->napi);
2064                         qlcnic_enable_tx_intr(adapter, tx_ring);
2065                 }
2066         }
2067 }
2068
2069 void qlcnic_83xx_napi_disable(struct qlcnic_adapter *adapter)
2070 {
2071         int ring;
2072         struct qlcnic_host_sds_ring *sds_ring;
2073         struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
2074         struct qlcnic_host_tx_ring *tx_ring;
2075
2076         if (adapter->is_up != QLCNIC_ADAPTER_UP_MAGIC)
2077                 return;
2078
2079         for (ring = 0; ring < adapter->drv_sds_rings; ring++) {
2080                 sds_ring = &recv_ctx->sds_rings[ring];
2081                 if (adapter->flags & QLCNIC_MSIX_ENABLED)
2082                         qlcnic_disable_sds_intr(adapter, sds_ring);
2083                 napi_synchronize(&sds_ring->napi);
2084                 napi_disable(&sds_ring->napi);
2085         }
2086
2087         if ((adapter->flags & QLCNIC_MSIX_ENABLED) &&
2088             !(adapter->flags & QLCNIC_TX_INTR_SHARED)) {
2089                 for (ring = 0; ring < adapter->drv_tx_rings; ring++) {
2090                         tx_ring = &adapter->tx_ring[ring];
2091                         qlcnic_disable_tx_intr(adapter, tx_ring);
2092                         napi_synchronize(&tx_ring->napi);
2093                         napi_disable(&tx_ring->napi);
2094                 }
2095         }
2096 }
2097
2098 int qlcnic_83xx_napi_add(struct qlcnic_adapter *adapter,
2099                          struct net_device *netdev)
2100 {
2101         int ring;
2102         struct qlcnic_host_sds_ring *sds_ring;
2103         struct qlcnic_host_tx_ring *tx_ring;
2104         struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
2105
2106         if (qlcnic_alloc_sds_rings(recv_ctx, adapter->drv_sds_rings))
2107                 return -ENOMEM;
2108
2109         for (ring = 0; ring < adapter->drv_sds_rings; ring++) {
2110                 sds_ring = &recv_ctx->sds_rings[ring];
2111                 if (adapter->flags & QLCNIC_MSIX_ENABLED) {
2112                         if (!(adapter->flags & QLCNIC_TX_INTR_SHARED))
2113                                 netif_napi_add(netdev, &sds_ring->napi,
2114                                                qlcnic_83xx_rx_poll,
2115                                                NAPI_POLL_WEIGHT);
2116                         else
2117                                 netif_napi_add(netdev, &sds_ring->napi,
2118                                                qlcnic_83xx_msix_sriov_vf_poll,
2119                                                NAPI_POLL_WEIGHT);
2120
2121                 } else {
2122                         netif_napi_add(netdev, &sds_ring->napi,
2123                                        qlcnic_83xx_poll,
2124                                        NAPI_POLL_WEIGHT);
2125                 }
2126         }
2127
2128         if (qlcnic_alloc_tx_rings(adapter, netdev)) {
2129                 qlcnic_free_sds_rings(recv_ctx);
2130                 return -ENOMEM;
2131         }
2132
2133         if ((adapter->flags & QLCNIC_MSIX_ENABLED) &&
2134             !(adapter->flags & QLCNIC_TX_INTR_SHARED)) {
2135                 for (ring = 0; ring < adapter->drv_tx_rings; ring++) {
2136                         tx_ring = &adapter->tx_ring[ring];
2137                         netif_napi_add(netdev, &tx_ring->napi,
2138                                        qlcnic_83xx_msix_tx_poll,
2139                                        NAPI_POLL_WEIGHT);
2140                 }
2141         }
2142
2143         return 0;
2144 }
2145
2146 void qlcnic_83xx_napi_del(struct qlcnic_adapter *adapter)
2147 {
2148         int ring;
2149         struct qlcnic_host_sds_ring *sds_ring;
2150         struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
2151         struct qlcnic_host_tx_ring *tx_ring;
2152
2153         for (ring = 0; ring < adapter->drv_sds_rings; ring++) {
2154                 sds_ring = &recv_ctx->sds_rings[ring];
2155                 netif_napi_del(&sds_ring->napi);
2156         }
2157
2158         qlcnic_free_sds_rings(adapter->recv_ctx);
2159
2160         if ((adapter->flags & QLCNIC_MSIX_ENABLED) &&
2161             !(adapter->flags & QLCNIC_TX_INTR_SHARED)) {
2162                 for (ring = 0; ring < adapter->drv_tx_rings; ring++) {
2163                         tx_ring = &adapter->tx_ring[ring];
2164                         netif_napi_del(&tx_ring->napi);
2165                 }
2166         }
2167
2168         qlcnic_free_tx_rings(adapter);
2169 }
2170
2171 static void qlcnic_83xx_process_rcv_diag(struct qlcnic_adapter *adapter,
2172                                          int ring, u64 sts_data[])
2173 {
2174         struct qlcnic_recv_context *recv_ctx = adapter->recv_ctx;
2175         struct sk_buff *skb;
2176         struct qlcnic_host_rds_ring *rds_ring;
2177         int index, length;
2178
2179         if (unlikely(ring >= adapter->max_rds_rings))
2180                 return;
2181
2182         rds_ring = &recv_ctx->rds_rings[ring];
2183         index = qlcnic_83xx_hndl(sts_data[0]);
2184         if (unlikely(index >= rds_ring->num_desc))
2185                 return;
2186
2187         length = qlcnic_83xx_pktln(sts_data[0]);
2188
2189         skb = qlcnic_process_rxbuf(adapter, rds_ring, index, STATUS_CKSUM_OK);
2190         if (!skb)
2191                 return;
2192
2193         if (length > rds_ring->skb_size)
2194                 skb_put(skb, rds_ring->skb_size);
2195         else
2196                 skb_put(skb, length);
2197
2198         if (!qlcnic_check_loopback_buff(skb->data, adapter->mac_addr))
2199                 adapter->ahw->diag_cnt++;
2200         else
2201                 dump_skb(skb, adapter);
2202
2203         dev_kfree_skb_any(skb);
2204         return;
2205 }
2206
2207 void qlcnic_83xx_process_rcv_ring_diag(struct qlcnic_host_sds_ring *sds_ring)
2208 {
2209         struct qlcnic_adapter *adapter = sds_ring->adapter;
2210         struct status_desc *desc;
2211         u64 sts_data[2];
2212         int ring, opcode;
2213         u32 consumer = sds_ring->consumer;
2214
2215         desc = &sds_ring->desc_head[consumer];
2216         sts_data[0] = le64_to_cpu(desc->status_desc_data[0]);
2217         sts_data[1] = le64_to_cpu(desc->status_desc_data[1]);
2218         opcode = qlcnic_83xx_opcode(sts_data[1]);
2219         if (!opcode)
2220                 return;
2221
2222         ring = QLCNIC_FETCH_RING_ID(qlcnic_83xx_hndl(sts_data[0]));
2223         qlcnic_83xx_process_rcv_diag(adapter, ring, sts_data);
2224         desc = &sds_ring->desc_head[consumer];
2225         desc->status_desc_data[0] = cpu_to_le64(STATUS_OWNER_PHANTOM);
2226         consumer = get_next_index(consumer, sds_ring->num_desc);
2227         sds_ring->consumer = consumer;
2228         writel(consumer, sds_ring->crb_sts_consumer);
2229 }