drivers/net/ethernet/emulex/benet/be_main.c

   1 /*
   2  * Copyright (C) 2005 - 2014 Emulex
   3  * All rights reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License version 2
   7  * as published by the Free Software Foundation.  The full GNU General
   8  * Public License is included in this distribution in the file called COPYING.
   9  *
  10  * Contact Information:
  11  * linux-drivers@emulex.com
  12  *
  13  * Emulex
  14  * 3333 Susan Street
  15  * Costa Mesa, CA 92626
  16  */
  17
  18 #include <linux/prefetch.h>
  19 #include <linux/module.h>
  20 #include "be.h"
  21 #include "be_cmds.h"
  22 #include <asm/div64.h>
  23 #include <linux/aer.h>
  24 #include <linux/if_bridge.h>
  25 #include <net/busy_poll.h>
  26 #include <net/vxlan.h>
  27
  28 MODULE_VERSION(DRV_VER);
  29 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  30 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER);
  31 MODULE_AUTHOR("Emulex Corporation");
  32 MODULE_LICENSE("GPL");
  33
  34 static unsigned int num_vfs;
  35 module_param(num_vfs, uint, S_IRUGO);
  36 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize");
  37
  38 static ushort rx_frag_size = 2048;
  39 module_param(rx_frag_size, ushort, S_IRUGO);
  40 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data.");
  41
  42 static const struct pci_device_id be_dev_ids[] = {
  43         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) },
  44         { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) },
  45         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) },
  46         { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) },
  47         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)},
  48         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)},
  49         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)},
  50         { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)},
  51         { 0 }
  52 };
  53 MODULE_DEVICE_TABLE(pci, be_dev_ids);
  54 /* UE Status Low CSR */
  55 static const char * const ue_status_low_desc[] = {
  56         "CEV",
  57         "CTX",
  58         "DBUF",
  59         "ERX",
  60         "Host",
  61         "MPU",
  62         "NDMA",
  63         "PTC ",
  64         "RDMA ",
  65         "RXF ",
  66         "RXIPS ",
  67         "RXULP0 ",
  68         "RXULP1 ",
  69         "RXULP2 ",
  70         "TIM ",
  71         "TPOST ",
  72         "TPRE ",
  73         "TXIPS ",
  74         "TXULP0 ",
  75         "TXULP1 ",
  76         "UC ",
  77         "WDMA ",
  78         "TXULP2 ",
  79         "HOST1 ",
  80         "P0_OB_LINK ",
  81         "P1_OB_LINK ",
  82         "HOST_GPIO ",
  83         "MBOX ",
  84         "ERX2 ",
  85         "SPARE ",
  86         "JTAG ",
  87         "MPU_INTPEND "
  88 };
  89
  90 /* UE Status High CSR */
  91 static const char * const ue_status_hi_desc[] = {
  92         "LPCMEMHOST",
  93         "MGMT_MAC",
  94         "PCS0ONLINE",
  95         "MPU_IRAM",
  96         "PCS1ONLINE",
  97         "PCTL0",
  98         "PCTL1",
  99         "PMEM",
 100         "RR",
 101         "TXPB",
 102         "RXPP",
 103         "XAUI",
 104         "TXP",
 105         "ARM",
 106         "IPC",
 107         "HOST2",
 108         "HOST3",
 109         "HOST4",
 110         "HOST5",
 111         "HOST6",
 112         "HOST7",
 113         "ECRC",
 114         "Poison TLP",
 115         "NETC",
 116         "PERIPH",
 117         "LLTXULP",
 118         "D2P",
 119         "RCON",
 120         "LDMA",
 121         "LLTXP",
 122         "LLTXPB",
 123         "Unknown"
 124 };
 125
 126 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q)
 127 {
 128         struct be_dma_mem *mem = &q->dma_mem;
 129
 130         if (mem->va) {
 131                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
 132                                   mem->dma);
 133                 mem->va = NULL;
 134         }
 135 }
 136
 137 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q,
 138                           u16 len, u16 entry_size)
 139 {
 140         struct be_dma_mem *mem = &q->dma_mem;
 141
 142         memset(q, 0, sizeof(*q));
 143         q->len = len;
 144         q->entry_size = entry_size;
 145         mem->size = len * entry_size;
 146         mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma,
 147                                       GFP_KERNEL);
 148         if (!mem->va)
 149                 return -ENOMEM;
 150         return 0;
 151 }
 152
 153 static void be_reg_intr_set(struct be_adapter *adapter, bool enable)
 154 {
 155         u32 reg, enabled;
 156
 157         pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET,
 158                               &reg);
 159         enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 160
 161         if (!enabled && enable)
 162                 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 163         else if (enabled && !enable)
 164                 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK;
 165         else
 166                 return;
 167
 168         pci_write_config_dword(adapter->pdev,
 169                                PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg);
 170 }
 171
 172 static void be_intr_set(struct be_adapter *adapter, bool enable)
 173 {
 174         int status = 0;
 175
 176         /* On lancer interrupts can't be controlled via this register */
 177         if (lancer_chip(adapter))
 178                 return;
 179
 180         if (adapter->eeh_error)
 181                 return;
 182
 183         status = be_cmd_intr_set(adapter, enable);
 184         if (status)
 185                 be_reg_intr_set(adapter, enable);
 186 }
 187
 188 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted)
 189 {
 190         u32 val = 0;
 191
 192         val |= qid & DB_RQ_RING_ID_MASK;
 193         val |= posted << DB_RQ_NUM_POSTED_SHIFT;
 194
 195         wmb();
 196         iowrite32(val, adapter->db + DB_RQ_OFFSET);
 197 }
 198
 199 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo,
 200                           u16 posted)
 201 {
 202         u32 val = 0;
 203
 204         val |= txo->q.id & DB_TXULP_RING_ID_MASK;
 205         val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT;
 206
 207         wmb();
 208         iowrite32(val, adapter->db + txo->db_offset);
 209 }
 210
 211 static void be_eq_notify(struct be_adapter *adapter, u16 qid,
 212                          bool arm, bool clear_int, u16 num_popped)
 213 {
 214         u32 val = 0;
 215
 216         val |= qid & DB_EQ_RING_ID_MASK;
 217         val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT);
 218
 219         if (adapter->eeh_error)
 220                 return;
 221
 222         if (arm)
 223                 val |= 1 << DB_EQ_REARM_SHIFT;
 224         if (clear_int)
 225                 val |= 1 << DB_EQ_CLR_SHIFT;
 226         val |= 1 << DB_EQ_EVNT_SHIFT;
 227         val |= num_popped << DB_EQ_NUM_POPPED_SHIFT;
 228         iowrite32(val, adapter->db + DB_EQ_OFFSET);
 229 }
 230
 231 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped)
 232 {
 233         u32 val = 0;
 234
 235         val |= qid & DB_CQ_RING_ID_MASK;
 236         val |= ((qid & DB_CQ_RING_ID_EXT_MASK) <<
 237                         DB_CQ_RING_ID_EXT_MASK_SHIFT);
 238
 239         if (adapter->eeh_error)
 240                 return;
 241
 242         if (arm)
 243                 val |= 1 << DB_CQ_REARM_SHIFT;
 244         val |= num_popped << DB_CQ_NUM_POPPED_SHIFT;
 245         iowrite32(val, adapter->db + DB_CQ_OFFSET);
 246 }
 247
 248 static int be_mac_addr_set(struct net_device *netdev, void *p)
 249 {
 250         struct be_adapter *adapter = netdev_priv(netdev);
 251         struct device *dev = &adapter->pdev->dev;
 252         struct sockaddr *addr = p;
 253         int status;
 254         u8 mac[ETH_ALEN];
 255         u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0;
 256
 257         if (!is_valid_ether_addr(addr->sa_data))
 258                 return -EADDRNOTAVAIL;
 259
 260         /* Proceed further only if, User provided MAC is different
 261          * from active MAC
 262          */
 263         if (ether_addr_equal(addr->sa_data, netdev->dev_addr))
 264                 return 0;
 265
 266         /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT
 267          * privilege or if PF did not provision the new MAC address.
 268          * On BE3, this cmd will always fail if the VF doesn't have the
 269          * FILTMGMT privilege. This failure is OK, only if the PF programmed
 270          * the MAC for the VF.
 271          */
 272         status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data,
 273                                  adapter->if_handle, &adapter->pmac_id[0], 0);
 274         if (!status) {
 275                 curr_pmac_id = adapter->pmac_id[0];
 276
 277                 /* Delete the old programmed MAC. This call may fail if the
 278                  * old MAC was already deleted by the PF driver.
 279                  */
 280                 if (adapter->pmac_id[0] != old_pmac_id)
 281                         be_cmd_pmac_del(adapter, adapter->if_handle,
 282                                         old_pmac_id, 0);
 283         }
 284
 285         /* Decide if the new MAC is successfully activated only after
 286          * querying the FW
 287          */
 288         status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac,
 289                                        adapter->if_handle, true, 0);
 290         if (status)
 291                 goto err;
 292
 293         /* The MAC change did not happen, either due to lack of privilege
 294          * or PF didn't pre-provision.
 295          */
 296         if (!ether_addr_equal(addr->sa_data, mac)) {
 297                 status = -EPERM;
 298                 goto err;
 299         }
 300
 301         memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
 302         dev_info(dev, "MAC address changed to %pM\n", mac);
 303         return 0;
 304 err:
 305         dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data);
 306         return status;
 307 }
 308
 309 /* BE2 supports only v0 cmd */
 310 static void *hw_stats_from_cmd(struct be_adapter *adapter)
 311 {
 312         if (BE2_chip(adapter)) {
 313                 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va;
 314
 315                 return &cmd->hw_stats;
 316         } else if (BE3_chip(adapter)) {
 317                 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va;
 318
 319                 return &cmd->hw_stats;
 320         } else {
 321                 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va;
 322
 323                 return &cmd->hw_stats;
 324         }
 325 }
 326
 327 /* BE2 supports only v0 cmd */
 328 static void *be_erx_stats_from_cmd(struct be_adapter *adapter)
 329 {
 330         if (BE2_chip(adapter)) {
 331                 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 332
 333                 return &hw_stats->erx;
 334         } else if (BE3_chip(adapter)) {
 335                 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 336
 337                 return &hw_stats->erx;
 338         } else {
 339                 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 340
 341                 return &hw_stats->erx;
 342         }
 343 }
 344
 345 static void populate_be_v0_stats(struct be_adapter *adapter)
 346 {
 347         struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter);
 348         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 349         struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf;
 350         struct be_port_rxf_stats_v0 *port_stats =
 351                                         &rxf_stats->port[adapter->port_num];
 352         struct be_drv_stats *drvs = &adapter->drv_stats;
 353
 354         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 355         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 356         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 357         drvs->rx_control_frames = port_stats->rx_control_frames;
 358         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 359         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 360         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 361         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 362         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 363         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 364         drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow;
 365         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 366         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 367         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 368         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 369         drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow;
 370         drvs->rx_dropped_header_too_small =
 371                 port_stats->rx_dropped_header_too_small;
 372         drvs->rx_address_filtered =
 373                                         port_stats->rx_address_filtered +
 374                                         port_stats->rx_vlan_filtered;
 375         drvs->rx_alignment_symbol_errors =
 376                 port_stats->rx_alignment_symbol_errors;
 377
 378         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 379         drvs->tx_controlframes = port_stats->tx_controlframes;
 380
 381         if (adapter->port_num)
 382                 drvs->jabber_events = rxf_stats->port1_jabber_events;
 383         else
 384                 drvs->jabber_events = rxf_stats->port0_jabber_events;
 385         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 386         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 387         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 388         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 389         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 390         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 391         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 392 }
 393
 394 static void populate_be_v1_stats(struct be_adapter *adapter)
 395 {
 396         struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter);
 397         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 398         struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf;
 399         struct be_port_rxf_stats_v1 *port_stats =
 400                                         &rxf_stats->port[adapter->port_num];
 401         struct be_drv_stats *drvs = &adapter->drv_stats;
 402
 403         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 404         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 405         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 406         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 407         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 408         drvs->rx_control_frames = port_stats->rx_control_frames;
 409         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 410         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 411         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 412         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 413         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 414         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 415         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 416         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 417         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 418         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 419         drvs->rx_dropped_header_too_small =
 420                 port_stats->rx_dropped_header_too_small;
 421         drvs->rx_input_fifo_overflow_drop =
 422                 port_stats->rx_input_fifo_overflow_drop;
 423         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 424         drvs->rx_alignment_symbol_errors =
 425                 port_stats->rx_alignment_symbol_errors;
 426         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 427         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 428         drvs->tx_controlframes = port_stats->tx_controlframes;
 429         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 430         drvs->jabber_events = port_stats->jabber_events;
 431         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 432         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 433         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 434         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 435         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 436         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 437         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 438 }
 439
 440 static void populate_be_v2_stats(struct be_adapter *adapter)
 441 {
 442         struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter);
 443         struct be_pmem_stats *pmem_sts = &hw_stats->pmem;
 444         struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf;
 445         struct be_port_rxf_stats_v2 *port_stats =
 446                                         &rxf_stats->port[adapter->port_num];
 447         struct be_drv_stats *drvs = &adapter->drv_stats;
 448
 449         be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats));
 450         drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop;
 451         drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames;
 452         drvs->rx_pause_frames = port_stats->rx_pause_frames;
 453         drvs->rx_crc_errors = port_stats->rx_crc_errors;
 454         drvs->rx_control_frames = port_stats->rx_control_frames;
 455         drvs->rx_in_range_errors = port_stats->rx_in_range_errors;
 456         drvs->rx_frame_too_long = port_stats->rx_frame_too_long;
 457         drvs->rx_dropped_runt = port_stats->rx_dropped_runt;
 458         drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs;
 459         drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs;
 460         drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs;
 461         drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length;
 462         drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small;
 463         drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short;
 464         drvs->rx_out_range_errors = port_stats->rx_out_range_errors;
 465         drvs->rx_dropped_header_too_small =
 466                 port_stats->rx_dropped_header_too_small;
 467         drvs->rx_input_fifo_overflow_drop =
 468                 port_stats->rx_input_fifo_overflow_drop;
 469         drvs->rx_address_filtered = port_stats->rx_address_filtered;
 470         drvs->rx_alignment_symbol_errors =
 471                 port_stats->rx_alignment_symbol_errors;
 472         drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop;
 473         drvs->tx_pauseframes = port_stats->tx_pauseframes;
 474         drvs->tx_controlframes = port_stats->tx_controlframes;
 475         drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes;
 476         drvs->jabber_events = port_stats->jabber_events;
 477         drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf;
 478         drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr;
 479         drvs->forwarded_packets = rxf_stats->forwarded_packets;
 480         drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu;
 481         drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr;
 482         drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags;
 483         adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops;
 484         if (be_roce_supported(adapter)) {
 485                 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd;
 486                 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd;
 487                 drvs->rx_roce_frames = port_stats->roce_frames_received;
 488                 drvs->roce_drops_crc = port_stats->roce_drops_crc;
 489                 drvs->roce_drops_payload_len =
 490                         port_stats->roce_drops_payload_len;
 491         }
 492 }
 493
 494 static void populate_lancer_stats(struct be_adapter *adapter)
 495 {
 496         struct be_drv_stats *drvs = &adapter->drv_stats;
 497         struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter);
 498
 499         be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats));
 500         drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo;
 501         drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo;
 502         drvs->rx_control_frames = pport_stats->rx_control_frames_lo;
 503         drvs->rx_in_range_errors = pport_stats->rx_in_range_errors;
 504         drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo;
 505         drvs->rx_dropped_runt = pport_stats->rx_dropped_runt;
 506         drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors;
 507         drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors;
 508         drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors;
 509         drvs->rx_dropped_tcp_length =
 510                                 pport_stats->rx_dropped_invalid_tcp_length;
 511         drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small;
 512         drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short;
 513         drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors;
 514         drvs->rx_dropped_header_too_small =
 515                                 pport_stats->rx_dropped_header_too_small;
 516         drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 517         drvs->rx_address_filtered =
 518                                         pport_stats->rx_address_filtered +
 519                                         pport_stats->rx_vlan_filtered;
 520         drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo;
 521         drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow;
 522         drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo;
 523         drvs->tx_controlframes = pport_stats->tx_control_frames_lo;
 524         drvs->jabber_events = pport_stats->rx_jabbers;
 525         drvs->forwarded_packets = pport_stats->num_forwards_lo;
 526         drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo;
 527         drvs->rx_drops_too_many_frags =
 528                                 pport_stats->rx_drops_too_many_frags_lo;
 529 }
 530
 531 static void accumulate_16bit_val(u32 *acc, u16 val)
 532 {
 533 #define lo(x)                   (x & 0xFFFF)
 534 #define hi(x)                   (x & 0xFFFF0000)
 535         bool wrapped = val < lo(*acc);
 536         u32 newacc = hi(*acc) + val;
 537
 538         if (wrapped)
 539                 newacc += 65536;
 540         ACCESS_ONCE(*acc) = newacc;
 541 }
 542
 543 static void populate_erx_stats(struct be_adapter *adapter,
 544                                struct be_rx_obj *rxo, u32 erx_stat)
 545 {
 546         if (!BEx_chip(adapter))
 547                 rx_stats(rxo)->rx_drops_no_frags = erx_stat;
 548         else
 549                 /* below erx HW counter can actually wrap around after
 550                  * 65535. Driver accumulates a 32-bit value
 551                  */
 552                 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags,
 553                                      (u16)erx_stat);
 554 }
 555
 556 void be_parse_stats(struct be_adapter *adapter)
 557 {
 558         struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter);
 559         struct be_rx_obj *rxo;
 560         int i;
 561         u32 erx_stat;
 562
 563         if (lancer_chip(adapter)) {
 564                 populate_lancer_stats(adapter);
 565         } else {
 566                 if (BE2_chip(adapter))
 567                         populate_be_v0_stats(adapter);
 568                 else if (BE3_chip(adapter))
 569                         /* for BE3 */
 570                         populate_be_v1_stats(adapter);
 571                 else
 572                         populate_be_v2_stats(adapter);
 573
 574                 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */
 575                 for_all_rx_queues(adapter, rxo, i) {
 576                         erx_stat = erx->rx_drops_no_fragments[rxo->q.id];
 577                         populate_erx_stats(adapter, rxo, erx_stat);
 578                 }
 579         }
 580 }
 581
 582 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev,
 583                                                 struct rtnl_link_stats64 *stats)
 584 {
 585         struct be_adapter *adapter = netdev_priv(netdev);
 586         struct be_drv_stats *drvs = &adapter->drv_stats;
 587         struct be_rx_obj *rxo;
 588         struct be_tx_obj *txo;
 589         u64 pkts, bytes;
 590         unsigned int start;
 591         int i;
 592
 593         for_all_rx_queues(adapter, rxo, i) {
 594                 const struct be_rx_stats *rx_stats = rx_stats(rxo);
 595
 596                 do {
 597                         start = u64_stats_fetch_begin_irq(&rx_stats->sync);
 598                         pkts = rx_stats(rxo)->rx_pkts;
 599                         bytes = rx_stats(rxo)->rx_bytes;
 600                 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start));
 601                 stats->rx_packets += pkts;
 602                 stats->rx_bytes += bytes;
 603                 stats->multicast += rx_stats(rxo)->rx_mcast_pkts;
 604                 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs +
 605                                         rx_stats(rxo)->rx_drops_no_frags;
 606         }
 607
 608         for_all_tx_queues(adapter, txo, i) {
 609                 const struct be_tx_stats *tx_stats = tx_stats(txo);
 610
 611                 do {
 612                         start = u64_stats_fetch_begin_irq(&tx_stats->sync);
 613                         pkts = tx_stats(txo)->tx_pkts;
 614                         bytes = tx_stats(txo)->tx_bytes;
 615                 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start));
 616                 stats->tx_packets += pkts;
 617                 stats->tx_bytes += bytes;
 618         }
 619
 620         /* bad pkts received */
 621         stats->rx_errors = drvs->rx_crc_errors +
 622                 drvs->rx_alignment_symbol_errors +
 623                 drvs->rx_in_range_errors +
 624                 drvs->rx_out_range_errors +
 625                 drvs->rx_frame_too_long +
 626                 drvs->rx_dropped_too_small +
 627                 drvs->rx_dropped_too_short +
 628                 drvs->rx_dropped_header_too_small +
 629                 drvs->rx_dropped_tcp_length +
 630                 drvs->rx_dropped_runt;
 631
 632         /* detailed rx errors */
 633         stats->rx_length_errors = drvs->rx_in_range_errors +
 634                 drvs->rx_out_range_errors +
 635                 drvs->rx_frame_too_long;
 636
 637         stats->rx_crc_errors = drvs->rx_crc_errors;
 638
 639         /* frame alignment errors */
 640         stats->rx_frame_errors = drvs->rx_alignment_symbol_errors;
 641
 642         /* receiver fifo overrun */
 643         /* drops_no_pbuf is no per i/f, it's per BE card */
 644         stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop +
 645                                 drvs->rx_input_fifo_overflow_drop +
 646                                 drvs->rx_drops_no_pbuf;
 647         return stats;
 648 }
 649
 650 void be_link_status_update(struct be_adapter *adapter, u8 link_status)
 651 {
 652         struct net_device *netdev = adapter->netdev;
 653
 654         if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) {
 655                 netif_carrier_off(netdev);
 656                 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT;
 657         }
 658
 659         if (link_status)
 660                 netif_carrier_on(netdev);
 661         else
 662                 netif_carrier_off(netdev);
 663 }
 664
 665 static void be_tx_stats_update(struct be_tx_obj *txo,
 666                                u32 wrb_cnt, u32 copied, u32 gso_segs,
 667                                bool stopped)
 668 {
 669         struct be_tx_stats *stats = tx_stats(txo);
 670
 671         u64_stats_update_begin(&stats->sync);
 672         stats->tx_reqs++;
 673         stats->tx_wrbs += wrb_cnt;
 674         stats->tx_bytes += copied;
 675         stats->tx_pkts += (gso_segs ? gso_segs : 1);
 676         if (stopped)
 677                 stats->tx_stops++;
 678         u64_stats_update_end(&stats->sync);
 679 }
 680
 681 /* Determine number of WRB entries needed to xmit data in an skb */
 682 static u32 wrb_cnt_for_skb(struct be_adapter *adapter, struct sk_buff *skb,
 683                            bool *dummy)
 684 {
 685         int cnt = (skb->len > skb->data_len);
 686
 687         cnt += skb_shinfo(skb)->nr_frags;
 688
 689         /* to account for hdr wrb */
 690         cnt++;
 691         if (lancer_chip(adapter) || !(cnt & 1)) {
 692                 *dummy = false;
 693         } else {
 694                 /* add a dummy to make it an even num */
 695                 cnt++;
 696                 *dummy = true;
 697         }
 698         BUG_ON(cnt > BE_MAX_TX_FRAG_COUNT);
 699         return cnt;
 700 }
 701
 702 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len)
 703 {
 704         wrb->frag_pa_hi = upper_32_bits(addr);
 705         wrb->frag_pa_lo = addr & 0xFFFFFFFF;
 706         wrb->frag_len = len & ETH_WRB_FRAG_LEN_MASK;
 707         wrb->rsvd0 = 0;
 708 }
 709
 710 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter,
 711                                      struct sk_buff *skb)
 712 {
 713         u8 vlan_prio;
 714         u16 vlan_tag;
 715
 716         vlan_tag = vlan_tx_tag_get(skb);
 717         vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
 718         /* If vlan priority provided by OS is NOT in available bmap */
 719         if (!(adapter->vlan_prio_bmap & (1 << vlan_prio)))
 720                 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) |
 721                                 adapter->recommended_prio;
 722
 723         return vlan_tag;
 724 }
 725
 726 /* Used only for IP tunnel packets */
 727 static u16 skb_inner_ip_proto(struct sk_buff *skb)
 728 {
 729         return (inner_ip_hdr(skb)->version == 4) ?
 730                 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr;
 731 }
 732
 733 static u16 skb_ip_proto(struct sk_buff *skb)
 734 {
 735         return (ip_hdr(skb)->version == 4) ?
 736                 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr;
 737 }
 738
 739 static void wrb_fill_hdr(struct be_adapter *adapter, struct be_eth_hdr_wrb *hdr,
 740                          struct sk_buff *skb, u32 wrb_cnt, u32 len,
 741                          bool skip_hw_vlan)
 742 {
 743         u16 vlan_tag, proto;
 744
 745         memset(hdr, 0, sizeof(*hdr));
 746
 747         SET_TX_WRB_HDR_BITS(crc, hdr, 1);
 748
 749         if (skb_is_gso(skb)) {
 750                 SET_TX_WRB_HDR_BITS(lso, hdr, 1);
 751                 SET_TX_WRB_HDR_BITS(lso_mss, hdr, skb_shinfo(skb)->gso_size);
 752                 if (skb_is_gso_v6(skb) && !lancer_chip(adapter))
 753                         SET_TX_WRB_HDR_BITS(lso6, hdr, 1);
 754         } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 755                 if (skb->encapsulation) {
 756                         SET_TX_WRB_HDR_BITS(ipcs, hdr, 1);
 757                         proto = skb_inner_ip_proto(skb);
 758                 } else {
 759                         proto = skb_ip_proto(skb);
 760                 }
 761                 if (proto == IPPROTO_TCP)
 762                         SET_TX_WRB_HDR_BITS(tcpcs, hdr, 1);
 763                 else if (proto == IPPROTO_UDP)
 764                         SET_TX_WRB_HDR_BITS(udpcs, hdr, 1);
 765         }
 766
 767         if (vlan_tx_tag_present(skb)) {
 768                 SET_TX_WRB_HDR_BITS(vlan, hdr, 1);
 769                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 770                 SET_TX_WRB_HDR_BITS(vlan_tag, hdr, vlan_tag);
 771         }
 772
 773         /* To skip HW VLAN tagging: evt = 1, compl = 0 */
 774         SET_TX_WRB_HDR_BITS(complete, hdr, !skip_hw_vlan);
 775         SET_TX_WRB_HDR_BITS(event, hdr, 1);
 776         SET_TX_WRB_HDR_BITS(num_wrb, hdr, wrb_cnt);
 777         SET_TX_WRB_HDR_BITS(len, hdr, len);
 778 }
 779
 780 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb,
 781                           bool unmap_single)
 782 {
 783         dma_addr_t dma;
 784
 785         be_dws_le_to_cpu(wrb, sizeof(*wrb));
 786
 787         dma = (u64)wrb->frag_pa_hi << 32 | (u64)wrb->frag_pa_lo;
 788         if (wrb->frag_len) {
 789                 if (unmap_single)
 790                         dma_unmap_single(dev, dma, wrb->frag_len,
 791                                          DMA_TO_DEVICE);
 792                 else
 793                         dma_unmap_page(dev, dma, wrb->frag_len, DMA_TO_DEVICE);
 794         }
 795 }
 796
 797 static int make_tx_wrbs(struct be_adapter *adapter, struct be_queue_info *txq,
 798                         struct sk_buff *skb, u32 wrb_cnt, bool dummy_wrb,
 799                         bool skip_hw_vlan)
 800 {
 801         dma_addr_t busaddr;
 802         int i, copied = 0;
 803         struct device *dev = &adapter->pdev->dev;
 804         struct sk_buff *first_skb = skb;
 805         struct be_eth_wrb *wrb;
 806         struct be_eth_hdr_wrb *hdr;
 807         bool map_single = false;
 808         u16 map_head;
 809
 810         hdr = queue_head_node(txq);
 811         queue_head_inc(txq);
 812         map_head = txq->head;
 813
 814         if (skb->len > skb->data_len) {
 815                 int len = skb_headlen(skb);
 816
 817                 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE);
 818                 if (dma_mapping_error(dev, busaddr))
 819                         goto dma_err;
 820                 map_single = true;
 821                 wrb = queue_head_node(txq);
 822                 wrb_fill(wrb, busaddr, len);
 823                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 824                 queue_head_inc(txq);
 825                 copied += len;
 826         }
 827
 828         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 829                 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i];
 830
 831                 busaddr = skb_frag_dma_map(dev, frag, 0,
 832                                            skb_frag_size(frag), DMA_TO_DEVICE);
 833                 if (dma_mapping_error(dev, busaddr))
 834                         goto dma_err;
 835                 wrb = queue_head_node(txq);
 836                 wrb_fill(wrb, busaddr, skb_frag_size(frag));
 837                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 838                 queue_head_inc(txq);
 839                 copied += skb_frag_size(frag);
 840         }
 841
 842         if (dummy_wrb) {
 843                 wrb = queue_head_node(txq);
 844                 wrb_fill(wrb, 0, 0);
 845                 be_dws_cpu_to_le(wrb, sizeof(*wrb));
 846                 queue_head_inc(txq);
 847         }
 848
 849         wrb_fill_hdr(adapter, hdr, first_skb, wrb_cnt, copied, skip_hw_vlan);
 850         be_dws_cpu_to_le(hdr, sizeof(*hdr));
 851
 852         return copied;
 853 dma_err:
 854         txq->head = map_head;
 855         while (copied) {
 856                 wrb = queue_head_node(txq);
 857                 unmap_tx_frag(dev, wrb, map_single);
 858                 map_single = false;
 859                 copied -= wrb->frag_len;
 860                 adapter->drv_stats.dma_map_errors++;
 861                 queue_head_inc(txq);
 862         }
 863         return 0;
 864 }
 865
 866 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter,
 867                                              struct sk_buff *skb,
 868                                              bool *skip_hw_vlan)
 869 {
 870         u16 vlan_tag = 0;
 871
 872         skb = skb_share_check(skb, GFP_ATOMIC);
 873         if (unlikely(!skb))
 874                 return skb;
 875
 876         if (vlan_tx_tag_present(skb))
 877                 vlan_tag = be_get_tx_vlan_tag(adapter, skb);
 878
 879         if (qnq_async_evt_rcvd(adapter) && adapter->pvid) {
 880                 if (!vlan_tag)
 881                         vlan_tag = adapter->pvid;
 882                 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to
 883                  * skip VLAN insertion
 884                  */
 885                 if (skip_hw_vlan)
 886                         *skip_hw_vlan = true;
 887         }
 888
 889         if (vlan_tag) {
 890                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 891                                                 vlan_tag);
 892                 if (unlikely(!skb))
 893                         return skb;
 894                 skb->vlan_tci = 0;
 895         }
 896
 897         /* Insert the outer VLAN, if any */
 898         if (adapter->qnq_vid) {
 899                 vlan_tag = adapter->qnq_vid;
 900                 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q),
 901                                                 vlan_tag);
 902                 if (unlikely(!skb))
 903                         return skb;
 904                 if (skip_hw_vlan)
 905                         *skip_hw_vlan = true;
 906         }
 907
 908         return skb;
 909 }
 910
 911 static bool be_ipv6_exthdr_check(struct sk_buff *skb)
 912 {
 913         struct ethhdr *eh = (struct ethhdr *)skb->data;
 914         u16 offset = ETH_HLEN;
 915
 916         if (eh->h_proto == htons(ETH_P_IPV6)) {
 917                 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset);
 918
 919                 offset += sizeof(struct ipv6hdr);
 920                 if (ip6h->nexthdr != NEXTHDR_TCP &&
 921                     ip6h->nexthdr != NEXTHDR_UDP) {
 922                         struct ipv6_opt_hdr *ehdr =
 923                                 (struct ipv6_opt_hdr *)(skb->data + offset);
 924
 925                         /* offending pkt: 2nd byte following IPv6 hdr is 0xff */
 926                         if (ehdr->hdrlen == 0xff)
 927                                 return true;
 928                 }
 929         }
 930         return false;
 931 }
 932
 933 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb)
 934 {
 935         return vlan_tx_tag_present(skb) || adapter->pvid || adapter->qnq_vid;
 936 }
 937
 938 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb)
 939 {
 940         return BE3_chip(adapter) && be_ipv6_exthdr_check(skb);
 941 }
 942
 943 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter,
 944                                                   struct sk_buff *skb,
 945                                                   bool *skip_hw_vlan)
 946 {
 947         struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
 948         unsigned int eth_hdr_len;
 949         struct iphdr *ip;
 950
 951         /* For padded packets, BE HW modifies tot_len field in IP header
 952          * incorrecly when VLAN tag is inserted by HW.
 953          * For padded packets, Lancer computes incorrect checksum.
 954          */
 955         eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ?
 956                                                 VLAN_ETH_HLEN : ETH_HLEN;
 957         if (skb->len <= 60 &&
 958             (lancer_chip(adapter) || vlan_tx_tag_present(skb)) &&
 959             is_ipv4_pkt(skb)) {
 960                 ip = (struct iphdr *)ip_hdr(skb);
 961                 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len));
 962         }
 963
 964         /* If vlan tag is already inlined in the packet, skip HW VLAN
 965          * tagging in pvid-tagging mode
 966          */
 967         if (be_pvid_tagging_enabled(adapter) &&
 968             veh->h_vlan_proto == htons(ETH_P_8021Q))
 969                 *skip_hw_vlan = true;
 970
 971         /* HW has a bug wherein it will calculate CSUM for VLAN
 972          * pkts even though it is disabled.
 973          * Manually insert VLAN in pkt.
 974          */
 975         if (skb->ip_summed != CHECKSUM_PARTIAL &&
 976             vlan_tx_tag_present(skb)) {
 977                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
 978                 if (unlikely(!skb))
 979                         goto err;
 980         }
 981
 982         /* HW may lockup when VLAN HW tagging is requested on
 983          * certain ipv6 packets. Drop such pkts if the HW workaround to
 984          * skip HW tagging is not enabled by FW.
 985          */
 986         if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) &&
 987                      (adapter->pvid || adapter->qnq_vid) &&
 988                      !qnq_async_evt_rcvd(adapter)))
 989                 goto tx_drop;
 990
 991         /* Manual VLAN tag insertion to prevent:
 992          * ASIC lockup when the ASIC inserts VLAN tag into
 993          * certain ipv6 packets. Insert VLAN tags in driver,
 994          * and set event, completion, vlan bits accordingly
 995          * in the Tx WRB.
 996          */
 997         if (be_ipv6_tx_stall_chk(adapter, skb) &&
 998             be_vlan_tag_tx_chk(adapter, skb)) {
 999                 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan);
1000                 if (unlikely(!skb))
1001                         goto err;
1002         }
1003
1004         return skb;
1005 tx_drop:
1006         dev_kfree_skb_any(skb);
1007 err:
1008         return NULL;
1009 }
1010
1011 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter,
1012                                            struct sk_buff *skb,
1013                                            bool *skip_hw_vlan)
1014 {
1015         /* Lancer, SH-R ASICs have a bug wherein Packets that are 32 bytes or
1016          * less may cause a transmit stall on that port. So the work-around is
1017          * to pad short packets (<= 32 bytes) to a 36-byte length.
1018          */
1019         if (unlikely(!BEx_chip(adapter) && skb->len <= 32)) {
1020                 if (skb_put_padto(skb, 36))
1021                         return NULL;
1022         }
1023
1024         if (BEx_chip(adapter) || lancer_chip(adapter)) {
1025                 skb = be_lancer_xmit_workarounds(adapter, skb, skip_hw_vlan);
1026                 if (!skb)
1027                         return NULL;
1028         }
1029
1030         return skb;
1031 }
1032
1033 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev)
1034 {
1035         struct be_adapter *adapter = netdev_priv(netdev);
1036         struct be_tx_obj *txo = &adapter->tx_obj[skb_get_queue_mapping(skb)];
1037         struct be_queue_info *txq = &txo->q;
1038         bool dummy_wrb, stopped = false;
1039         u32 wrb_cnt = 0, copied = 0;
1040         bool skip_hw_vlan = false;
1041         u32 start = txq->head;
1042
1043         skb = be_xmit_workarounds(adapter, skb, &skip_hw_vlan);
1044         if (!skb) {
1045                 tx_stats(txo)->tx_drv_drops++;
1046                 return NETDEV_TX_OK;
1047         }
1048
1049         wrb_cnt = wrb_cnt_for_skb(adapter, skb, &dummy_wrb);
1050
1051         copied = make_tx_wrbs(adapter, txq, skb, wrb_cnt, dummy_wrb,
1052                               skip_hw_vlan);
1053         if (copied) {
1054                 int gso_segs = skb_shinfo(skb)->gso_segs;
1055
1056                 /* record the sent skb in the sent_skb table */
1057                 BUG_ON(txo->sent_skb_list[start]);
1058                 txo->sent_skb_list[start] = skb;
1059
1060                 /* Ensure txq has space for the next skb; Else stop the queue
1061                  * *BEFORE* ringing the tx doorbell, so that we serialze the
1062                  * tx compls of the current transmit which'll wake up the queue
1063                  */
1064                 atomic_add(wrb_cnt, &txq->used);
1065                 if ((BE_MAX_TX_FRAG_COUNT + atomic_read(&txq->used)) >=
1066                                                                 txq->len) {
1067                         netif_stop_subqueue(netdev, skb_get_queue_mapping(skb));
1068                         stopped = true;
1069                 }
1070
1071                 be_txq_notify(adapter, txo, wrb_cnt);
1072
1073                 be_tx_stats_update(txo, wrb_cnt, copied, gso_segs, stopped);
1074         } else {
1075                 txq->head = start;
1076                 tx_stats(txo)->tx_drv_drops++;
1077                 dev_kfree_skb_any(skb);
1078         }
1079         return NETDEV_TX_OK;
1080 }
1081
1082 static int be_change_mtu(struct net_device *netdev, int new_mtu)
1083 {
1084         struct be_adapter *adapter = netdev_priv(netdev);
1085         struct device *dev = &adapter->pdev->dev;
1086
1087         if (new_mtu < BE_MIN_MTU || new_mtu > BE_MAX_MTU) {
1088                 dev_info(dev, "MTU must be between %d and %d bytes\n",
1089                          BE_MIN_MTU, BE_MAX_MTU);
1090                 return -EINVAL;
1091         }
1092
1093         dev_info(dev, "MTU changed from %d to %d bytes\n",
1094                  netdev->mtu, new_mtu);
1095         netdev->mtu = new_mtu;
1096         return 0;
1097 }
1098
1099 /*
1100  * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE.
1101  * If the user configures more, place BE in vlan promiscuous mode.
1102  */
1103 static int be_vid_config(struct be_adapter *adapter)
1104 {
1105         struct device *dev = &adapter->pdev->dev;
1106         u16 vids[BE_NUM_VLANS_SUPPORTED];
1107         u16 num = 0, i = 0;
1108         int status = 0;
1109
1110         /* No need to further configure vids if in promiscuous mode */
1111         if (adapter->promiscuous)
1112                 return 0;
1113
1114         if (adapter->vlans_added > be_max_vlans(adapter))
1115                 goto set_vlan_promisc;
1116
1117         /* Construct VLAN Table to give to HW */
1118         for_each_set_bit(i, adapter->vids, VLAN_N_VID)
1119                 vids[num++] = cpu_to_le16(i);
1120
1121         status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num);
1122         if (status) {
1123                 /* Set to VLAN promisc mode as setting VLAN filter failed */
1124                 if (addl_status(status) ==
1125                                 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES)
1126                         goto set_vlan_promisc;
1127                 dev_err(dev, "Setting HW VLAN filtering failed\n");
1128         } else {
1129                 if (adapter->flags & BE_FLAGS_VLAN_PROMISC) {
1130                         /* hw VLAN filtering re-enabled. */
1131                         status = be_cmd_rx_filter(adapter,
1132                                                   BE_FLAGS_VLAN_PROMISC, OFF);
1133                         if (!status) {
1134                                 dev_info(dev,
1135                                          "Disabling VLAN Promiscuous mode\n");
1136                                 adapter->flags &= ~BE_FLAGS_VLAN_PROMISC;
1137                         }
1138                 }
1139         }
1140
1141         return status;
1142
1143 set_vlan_promisc:
1144         if (adapter->flags & BE_FLAGS_VLAN_PROMISC)
1145                 return 0;
1146
1147         status = be_cmd_rx_filter(adapter, BE_FLAGS_VLAN_PROMISC, ON);
1148         if (!status) {
1149                 dev_info(dev, "Enable VLAN Promiscuous mode\n");
1150                 adapter->flags |= BE_FLAGS_VLAN_PROMISC;
1151         } else
1152                 dev_err(dev, "Failed to enable VLAN Promiscuous mode\n");
1153         return status;
1154 }
1155
1156 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
1157 {
1158         struct be_adapter *adapter = netdev_priv(netdev);
1159         int status = 0;
1160
1161         /* Packets with VID 0 are always received by Lancer by default */
1162         if (lancer_chip(adapter) && vid == 0)
1163                 return status;
1164
1165         if (test_bit(vid, adapter->vids))
1166                 return status;
1167
1168         set_bit(vid, adapter->vids);
1169         adapter->vlans_added++;
1170
1171         status = be_vid_config(adapter);
1172         if (status) {
1173                 adapter->vlans_added--;
1174                 clear_bit(vid, adapter->vids);
1175         }
1176
1177         return status;
1178 }
1179
1180 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid)
1181 {
1182         struct be_adapter *adapter = netdev_priv(netdev);
1183
1184         /* Packets with VID 0 are always received by Lancer by default */
1185         if (lancer_chip(adapter) && vid == 0)
1186                 return 0;
1187
1188         clear_bit(vid, adapter->vids);
1189         adapter->vlans_added--;
1190
1191         return be_vid_config(adapter);
1192 }
1193
1194 static void be_clear_promisc(struct be_adapter *adapter)
1195 {
1196         adapter->promiscuous = false;
1197         adapter->flags &= ~(BE_FLAGS_VLAN_PROMISC | BE_FLAGS_MCAST_PROMISC);
1198
1199         be_cmd_rx_filter(adapter, IFF_PROMISC, OFF);
1200 }
1201
1202 static void be_set_rx_mode(struct net_device *netdev)
1203 {
1204         struct be_adapter *adapter = netdev_priv(netdev);
1205         int status;
1206
1207         if (netdev->flags & IFF_PROMISC) {
1208                 be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1209                 adapter->promiscuous = true;
1210                 goto done;
1211         }
1212
1213         /* BE was previously in promiscuous mode; disable it */
1214         if (adapter->promiscuous) {
1215                 be_clear_promisc(adapter);
1216                 if (adapter->vlans_added)
1217                         be_vid_config(adapter);
1218         }
1219
1220         /* Enable multicast promisc if num configured exceeds what we support */
1221         if (netdev->flags & IFF_ALLMULTI ||
1222             netdev_mc_count(netdev) > be_max_mc(adapter))
1223                 goto set_mcast_promisc;
1224
1225         if (netdev_uc_count(netdev) != adapter->uc_macs) {
1226                 struct netdev_hw_addr *ha;
1227                 int i = 1; /* First slot is claimed by the Primary MAC */
1228
1229                 for (; adapter->uc_macs > 0; adapter->uc_macs--, i++) {
1230                         be_cmd_pmac_del(adapter, adapter->if_handle,
1231                                         adapter->pmac_id[i], 0);
1232                 }
1233
1234                 if (netdev_uc_count(netdev) > be_max_uc(adapter)) {
1235                         be_cmd_rx_filter(adapter, IFF_PROMISC, ON);
1236                         adapter->promiscuous = true;
1237                         goto done;
1238                 }
1239
1240                 netdev_for_each_uc_addr(ha, adapter->netdev) {
1241                         adapter->uc_macs++; /* First slot is for Primary MAC */
1242                         be_cmd_pmac_add(adapter, (u8 *)ha->addr,
1243                                         adapter->if_handle,
1244                                         &adapter->pmac_id[adapter->uc_macs], 0);
1245                 }
1246         }
1247
1248         status = be_cmd_rx_filter(adapter, IFF_MULTICAST, ON);
1249         if (!status) {
1250                 if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1251                         adapter->flags &= ~BE_FLAGS_MCAST_PROMISC;
1252                 goto done;
1253         }
1254
1255 set_mcast_promisc:
1256         if (adapter->flags & BE_FLAGS_MCAST_PROMISC)
1257                 return;
1258
1259         /* Set to MCAST promisc mode if setting MULTICAST address fails
1260          * or if num configured exceeds what we support
1261          */
1262         status = be_cmd_rx_filter(adapter, IFF_ALLMULTI, ON);
1263         if (!status)
1264                 adapter->flags |= BE_FLAGS_MCAST_PROMISC;
1265 done:
1266         return;
1267 }
1268
1269 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1270 {
1271         struct be_adapter *adapter = netdev_priv(netdev);
1272         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1273         int status;
1274
1275         if (!sriov_enabled(adapter))
1276                 return -EPERM;
1277
1278         if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs)
1279                 return -EINVAL;
1280
1281         /* Proceed further only if user provided MAC is different
1282          * from active MAC
1283          */
1284         if (ether_addr_equal(mac, vf_cfg->mac_addr))
1285                 return 0;
1286
1287         if (BEx_chip(adapter)) {
1288                 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id,
1289                                 vf + 1);
1290
1291                 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle,
1292                                          &vf_cfg->pmac_id, vf + 1);
1293         } else {
1294                 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
1295                                         vf + 1);
1296         }
1297
1298         if (status) {
1299                 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x",
1300                         mac, vf, status);
1301                 return be_cmd_status(status);
1302         }
1303
1304         ether_addr_copy(vf_cfg->mac_addr, mac);
1305
1306         return 0;
1307 }
1308
1309 static int be_get_vf_config(struct net_device *netdev, int vf,
1310                             struct ifla_vf_info *vi)
1311 {
1312         struct be_adapter *adapter = netdev_priv(netdev);
1313         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1314
1315         if (!sriov_enabled(adapter))
1316                 return -EPERM;
1317
1318         if (vf >= adapter->num_vfs)
1319                 return -EINVAL;
1320
1321         vi->vf = vf;
1322         vi->max_tx_rate = vf_cfg->tx_rate;
1323         vi->min_tx_rate = 0;
1324         vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK;
1325         vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT;
1326         memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN);
1327         vi->linkstate = adapter->vf_cfg[vf].plink_tracking;
1328
1329         return 0;
1330 }
1331
1332 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos)
1333 {
1334         struct be_adapter *adapter = netdev_priv(netdev);
1335         struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf];
1336         int status = 0;
1337
1338         if (!sriov_enabled(adapter))
1339                 return -EPERM;
1340
1341         if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7)
1342                 return -EINVAL;
1343
1344         if (vlan || qos) {
1345                 vlan |= qos << VLAN_PRIO_SHIFT;
1346                 if (vf_cfg->vlan_tag != vlan)
1347                         status = be_cmd_set_hsw_config(adapter, vlan, vf + 1,
1348                                                        vf_cfg->if_handle, 0);
1349         } else {
1350                 /* Reset Transparent Vlan Tagging. */
1351                 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID,
1352                                                vf + 1, vf_cfg->if_handle, 0);
1353         }
1354
1355         if (status) {
1356                 dev_err(&adapter->pdev->dev,
1357                         "VLAN %d config on VF %d failed : %#x\n", vlan,
1358                         vf, status);
1359                 return be_cmd_status(status);
1360         }
1361
1362         vf_cfg->vlan_tag = vlan;
1363
1364         return 0;
1365 }
1366
1367 static int be_set_vf_tx_rate(struct net_device *netdev, int vf,
1368                              int min_tx_rate, int max_tx_rate)
1369 {
1370         struct be_adapter *adapter = netdev_priv(netdev);
1371         struct device *dev = &adapter->pdev->dev;
1372         int percent_rate, status = 0;
1373         u16 link_speed = 0;
1374         u8 link_status;
1375
1376         if (!sriov_enabled(adapter))
1377                 return -EPERM;
1378
1379         if (vf >= adapter->num_vfs)
1380                 return -EINVAL;
1381
1382         if (min_tx_rate)
1383                 return -EINVAL;
1384
1385         if (!max_tx_rate)
1386                 goto config_qos;
1387
1388         status = be_cmd_link_status_query(adapter, &link_speed,
1389                                           &link_status, 0);
1390         if (status)
1391                 goto err;
1392
1393         if (!link_status) {
1394                 dev_err(dev, "TX-rate setting not allowed when link is down\n");
1395                 status = -ENETDOWN;
1396                 goto err;
1397         }
1398
1399         if (max_tx_rate < 100 || max_tx_rate > link_speed) {
1400                 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n",
1401                         link_speed);
1402                 status = -EINVAL;
1403                 goto err;
1404         }
1405
1406         /* On Skyhawk the QOS setting must be done only as a % value */
1407         percent_rate = link_speed / 100;
1408         if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) {
1409                 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n",
1410                         percent_rate);
1411                 status = -EINVAL;
1412                 goto err;
1413         }
1414
1415 config_qos:
1416         status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1);
1417         if (status)
1418                 goto err;
1419
1420         adapter->vf_cfg[vf].tx_rate = max_tx_rate;
1421         return 0;
1422
1423 err:
1424         dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n",
1425                 max_tx_rate, vf);
1426         return be_cmd_status(status);
1427 }
1428
1429 static int be_set_vf_link_state(struct net_device *netdev, int vf,
1430                                 int link_state)
1431 {
1432         struct be_adapter *adapter = netdev_priv(netdev);
1433         int status;
1434
1435         if (!sriov_enabled(adapter))
1436                 return -EPERM;
1437
1438         if (vf >= adapter->num_vfs)
1439                 return -EINVAL;
1440
1441         status = be_cmd_set_logical_link_config(adapter, link_state, vf+1);
1442         if (status) {
1443                 dev_err(&adapter->pdev->dev,
1444                         "Link state change on VF %d failed: %#x\n", vf, status);
1445                 return be_cmd_status(status);
1446         }
1447
1448         adapter->vf_cfg[vf].plink_tracking = link_state;
1449
1450         return 0;
1451 }
1452
1453 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts,
1454                           ulong now)
1455 {
1456         aic->rx_pkts_prev = rx_pkts;
1457         aic->tx_reqs_prev = tx_pkts;
1458         aic->jiffies = now;
1459 }
1460
1461 static void be_eqd_update(struct be_adapter *adapter)
1462 {
1463         struct be_set_eqd set_eqd[MAX_EVT_QS];
1464         int eqd, i, num = 0, start;
1465         struct be_aic_obj *aic;
1466         struct be_eq_obj *eqo;
1467         struct be_rx_obj *rxo;
1468         struct be_tx_obj *txo;
1469         u64 rx_pkts, tx_pkts;
1470         ulong now;
1471         u32 pps, delta;
1472
1473         for_all_evt_queues(adapter, eqo, i) {
1474                 aic = &adapter->aic_obj[eqo->idx];
1475                 if (!aic->enable) {
1476                         if (aic->jiffies)
1477                                 aic->jiffies = 0;
1478                         eqd = aic->et_eqd;
1479                         goto modify_eqd;
1480                 }
1481
1482                 rxo = &adapter->rx_obj[eqo->idx];
1483                 do {
1484                         start = u64_stats_fetch_begin_irq(&rxo->stats.sync);
1485                         rx_pkts = rxo->stats.rx_pkts;
1486                 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start));
1487
1488                 txo = &adapter->tx_obj[eqo->idx];
1489                 do {
1490                         start = u64_stats_fetch_begin_irq(&txo->stats.sync);
1491                         tx_pkts = txo->stats.tx_reqs;
1492                 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start));
1493
1494                 /* Skip, if wrapped around or first calculation */
1495                 now = jiffies;
1496                 if (!aic->jiffies || time_before(now, aic->jiffies) ||
1497                     rx_pkts < aic->rx_pkts_prev ||
1498                     tx_pkts < aic->tx_reqs_prev) {
1499                         be_aic_update(aic, rx_pkts, tx_pkts, now);
1500                         continue;
1501                 }
1502
1503                 delta = jiffies_to_msecs(now - aic->jiffies);
1504                 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) +
1505                         (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta);
1506                 eqd = (pps / 15000) << 2;
1507
1508                 if (eqd < 8)
1509                         eqd = 0;
1510                 eqd = min_t(u32, eqd, aic->max_eqd);
1511                 eqd = max_t(u32, eqd, aic->min_eqd);
1512
1513                 be_aic_update(aic, rx_pkts, tx_pkts, now);
1514 modify_eqd:
1515                 if (eqd != aic->prev_eqd) {
1516                         set_eqd[num].delay_multiplier = (eqd * 65)/100;
1517                         set_eqd[num].eq_id = eqo->q.id;
1518                         aic->prev_eqd = eqd;
1519                         num++;
1520                 }
1521         }
1522
1523         if (num)
1524                 be_cmd_modify_eqd(adapter, set_eqd, num);
1525 }
1526
1527 static void be_rx_stats_update(struct be_rx_obj *rxo,
1528                                struct be_rx_compl_info *rxcp)
1529 {
1530         struct be_rx_stats *stats = rx_stats(rxo);
1531
1532         u64_stats_update_begin(&stats->sync);
1533         stats->rx_compl++;
1534         stats->rx_bytes += rxcp->pkt_size;
1535         stats->rx_pkts++;
1536         if (rxcp->pkt_type == BE_MULTICAST_PACKET)
1537                 stats->rx_mcast_pkts++;
1538         if (rxcp->err)
1539                 stats->rx_compl_err++;
1540         u64_stats_update_end(&stats->sync);
1541 }
1542
1543 static inline bool csum_passed(struct be_rx_compl_info *rxcp)
1544 {
1545         /* L4 checksum is not reliable for non TCP/UDP packets.
1546          * Also ignore ipcksm for ipv6 pkts
1547          */
1548         return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum &&
1549                 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err;
1550 }
1551
1552 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo)
1553 {
1554         struct be_adapter *adapter = rxo->adapter;
1555         struct be_rx_page_info *rx_page_info;
1556         struct be_queue_info *rxq = &rxo->q;
1557         u16 frag_idx = rxq->tail;
1558
1559         rx_page_info = &rxo->page_info_tbl[frag_idx];
1560         BUG_ON(!rx_page_info->page);
1561
1562         if (rx_page_info->last_frag) {
1563                 dma_unmap_page(&adapter->pdev->dev,
1564                                dma_unmap_addr(rx_page_info, bus),
1565                                adapter->big_page_size, DMA_FROM_DEVICE);
1566                 rx_page_info->last_frag = false;
1567         } else {
1568                 dma_sync_single_for_cpu(&adapter->pdev->dev,
1569                                         dma_unmap_addr(rx_page_info, bus),
1570                                         rx_frag_size, DMA_FROM_DEVICE);
1571         }
1572
1573         queue_tail_inc(rxq);
1574         atomic_dec(&rxq->used);
1575         return rx_page_info;
1576 }
1577
1578 /* Throwaway the data in the Rx completion */
1579 static void be_rx_compl_discard(struct be_rx_obj *rxo,
1580                                 struct be_rx_compl_info *rxcp)
1581 {
1582         struct be_rx_page_info *page_info;
1583         u16 i, num_rcvd = rxcp->num_rcvd;
1584
1585         for (i = 0; i < num_rcvd; i++) {
1586                 page_info = get_rx_page_info(rxo);
1587                 put_page(page_info->page);
1588                 memset(page_info, 0, sizeof(*page_info));
1589         }
1590 }
1591
1592 /*
1593  * skb_fill_rx_data forms a complete skb for an ether frame
1594  * indicated by rxcp.
1595  */
1596 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb,
1597                              struct be_rx_compl_info *rxcp)
1598 {
1599         struct be_rx_page_info *page_info;
1600         u16 i, j;
1601         u16 hdr_len, curr_frag_len, remaining;
1602         u8 *start;
1603
1604         page_info = get_rx_page_info(rxo);
1605         start = page_address(page_info->page) + page_info->page_offset;
1606         prefetch(start);
1607
1608         /* Copy data in the first descriptor of this completion */
1609         curr_frag_len = min(rxcp->pkt_size, rx_frag_size);
1610
1611         skb->len = curr_frag_len;
1612         if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */
1613                 memcpy(skb->data, start, curr_frag_len);
1614                 /* Complete packet has now been moved to data */
1615                 put_page(page_info->page);
1616                 skb->data_len = 0;
1617                 skb->tail += curr_frag_len;
1618         } else {
1619                 hdr_len = ETH_HLEN;
1620                 memcpy(skb->data, start, hdr_len);
1621                 skb_shinfo(skb)->nr_frags = 1;
1622                 skb_frag_set_page(skb, 0, page_info->page);
1623                 skb_shinfo(skb)->frags[0].page_offset =
1624                                         page_info->page_offset + hdr_len;
1625                 skb_frag_size_set(&skb_shinfo(skb)->frags[0],
1626                                   curr_frag_len - hdr_len);
1627                 skb->data_len = curr_frag_len - hdr_len;
1628                 skb->truesize += rx_frag_size;
1629                 skb->tail += hdr_len;
1630         }
1631         page_info->page = NULL;
1632
1633         if (rxcp->pkt_size <= rx_frag_size) {
1634                 BUG_ON(rxcp->num_rcvd != 1);
1635                 return;
1636         }
1637
1638         /* More frags present for this completion */
1639         remaining = rxcp->pkt_size - curr_frag_len;
1640         for (i = 1, j = 0; i < rxcp->num_rcvd; i++) {
1641                 page_info = get_rx_page_info(rxo);
1642                 curr_frag_len = min(remaining, rx_frag_size);
1643
1644                 /* Coalesce all frags from the same physical page in one slot */
1645                 if (page_info->page_offset == 0) {
1646                         /* Fresh page */
1647                         j++;
1648                         skb_frag_set_page(skb, j, page_info->page);
1649                         skb_shinfo(skb)->frags[j].page_offset =
1650                                                         page_info->page_offset;
1651                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1652                         skb_shinfo(skb)->nr_frags++;
1653                 } else {
1654                         put_page(page_info->page);
1655                 }
1656
1657                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1658                 skb->len += curr_frag_len;
1659                 skb->data_len += curr_frag_len;
1660                 skb->truesize += rx_frag_size;
1661                 remaining -= curr_frag_len;
1662                 page_info->page = NULL;
1663         }
1664         BUG_ON(j > MAX_SKB_FRAGS);
1665 }
1666
1667 /* Process the RX completion indicated by rxcp when GRO is disabled */
1668 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi,
1669                                 struct be_rx_compl_info *rxcp)
1670 {
1671         struct be_adapter *adapter = rxo->adapter;
1672         struct net_device *netdev = adapter->netdev;
1673         struct sk_buff *skb;
1674
1675         skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE);
1676         if (unlikely(!skb)) {
1677                 rx_stats(rxo)->rx_drops_no_skbs++;
1678                 be_rx_compl_discard(rxo, rxcp);
1679                 return;
1680         }
1681
1682         skb_fill_rx_data(rxo, skb, rxcp);
1683
1684         if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp)))
1685                 skb->ip_summed = CHECKSUM_UNNECESSARY;
1686         else
1687                 skb_checksum_none_assert(skb);
1688
1689         skb->protocol = eth_type_trans(skb, netdev);
1690         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1691         if (netdev->features & NETIF_F_RXHASH)
1692                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1693
1694         skb->csum_level = rxcp->tunneled;
1695         skb_mark_napi_id(skb, napi);
1696
1697         if (rxcp->vlanf)
1698                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1699
1700         netif_receive_skb(skb);
1701 }
1702
1703 /* Process the RX completion indicated by rxcp when GRO is enabled */
1704 static void be_rx_compl_process_gro(struct be_rx_obj *rxo,
1705                                     struct napi_struct *napi,
1706                                     struct be_rx_compl_info *rxcp)
1707 {
1708         struct be_adapter *adapter = rxo->adapter;
1709         struct be_rx_page_info *page_info;
1710         struct sk_buff *skb = NULL;
1711         u16 remaining, curr_frag_len;
1712         u16 i, j;
1713
1714         skb = napi_get_frags(napi);
1715         if (!skb) {
1716                 be_rx_compl_discard(rxo, rxcp);
1717                 return;
1718         }
1719
1720         remaining = rxcp->pkt_size;
1721         for (i = 0, j = -1; i < rxcp->num_rcvd; i++) {
1722                 page_info = get_rx_page_info(rxo);
1723
1724                 curr_frag_len = min(remaining, rx_frag_size);
1725
1726                 /* Coalesce all frags from the same physical page in one slot */
1727                 if (i == 0 || page_info->page_offset == 0) {
1728                         /* First frag or Fresh page */
1729                         j++;
1730                         skb_frag_set_page(skb, j, page_info->page);
1731                         skb_shinfo(skb)->frags[j].page_offset =
1732                                                         page_info->page_offset;
1733                         skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0);
1734                 } else {
1735                         put_page(page_info->page);
1736                 }
1737                 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len);
1738                 skb->truesize += rx_frag_size;
1739                 remaining -= curr_frag_len;
1740                 memset(page_info, 0, sizeof(*page_info));
1741         }
1742         BUG_ON(j > MAX_SKB_FRAGS);
1743
1744         skb_shinfo(skb)->nr_frags = j + 1;
1745         skb->len = rxcp->pkt_size;
1746         skb->data_len = rxcp->pkt_size;
1747         skb->ip_summed = CHECKSUM_UNNECESSARY;
1748         skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]);
1749         if (adapter->netdev->features & NETIF_F_RXHASH)
1750                 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3);
1751
1752         skb->csum_level = rxcp->tunneled;
1753         skb_mark_napi_id(skb, napi);
1754
1755         if (rxcp->vlanf)
1756                 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag);
1757
1758         napi_gro_frags(napi);
1759 }
1760
1761 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl,
1762                                  struct be_rx_compl_info *rxcp)
1763 {
1764         rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl);
1765         rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl);
1766         rxcp->err = GET_RX_COMPL_V1_BITS(err, compl);
1767         rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl);
1768         rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl);
1769         rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl);
1770         rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl);
1771         rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl);
1772         rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl);
1773         rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl);
1774         rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl);
1775         if (rxcp->vlanf) {
1776                 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl);
1777                 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl);
1778         }
1779         rxcp->port = GET_RX_COMPL_V1_BITS(port, compl);
1780         rxcp->tunneled =
1781                 GET_RX_COMPL_V1_BITS(tunneled, compl);
1782 }
1783
1784 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl,
1785                                  struct be_rx_compl_info *rxcp)
1786 {
1787         rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl);
1788         rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl);
1789         rxcp->err = GET_RX_COMPL_V0_BITS(err, compl);
1790         rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl);
1791         rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl);
1792         rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl);
1793         rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl);
1794         rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl);
1795         rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl);
1796         rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl);
1797         rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl);
1798         if (rxcp->vlanf) {
1799                 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl);
1800                 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl);
1801         }
1802         rxcp->port = GET_RX_COMPL_V0_BITS(port, compl);
1803         rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl);
1804 }
1805
1806 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo)
1807 {
1808         struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq);
1809         struct be_rx_compl_info *rxcp = &rxo->rxcp;
1810         struct be_adapter *adapter = rxo->adapter;
1811
1812         /* For checking the valid bit it is Ok to use either definition as the
1813          * valid bit is at the same position in both v0 and v1 Rx compl */
1814         if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0)
1815                 return NULL;
1816
1817         rmb();
1818         be_dws_le_to_cpu(compl, sizeof(*compl));
1819
1820         if (adapter->be3_native)
1821                 be_parse_rx_compl_v1(compl, rxcp);
1822         else
1823                 be_parse_rx_compl_v0(compl, rxcp);
1824
1825         if (rxcp->ip_frag)
1826                 rxcp->l4_csum = 0;
1827
1828         if (rxcp->vlanf) {
1829                 /* In QNQ modes, if qnq bit is not set, then the packet was
1830                  * tagged only with the transparent outer vlan-tag and must
1831                  * not be treated as a vlan packet by host
1832                  */
1833                 if (be_is_qnq_mode(adapter) && !rxcp->qnq)
1834                         rxcp->vlanf = 0;
1835
1836                 if (!lancer_chip(adapter))
1837                         rxcp->vlan_tag = swab16(rxcp->vlan_tag);
1838
1839                 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) &&
1840                     !test_bit(rxcp->vlan_tag, adapter->vids))
1841                         rxcp->vlanf = 0;
1842         }
1843
1844         /* As the compl has been parsed, reset it; we wont touch it again */
1845         compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0;
1846
1847         queue_tail_inc(&rxo->cq);
1848         return rxcp;
1849 }
1850
1851 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp)
1852 {
1853         u32 order = get_order(size);
1854
1855         if (order > 0)
1856                 gfp |= __GFP_COMP;
1857         return  alloc_pages(gfp, order);
1858 }
1859
1860 /*
1861  * Allocate a page, split it to fragments of size rx_frag_size and post as
1862  * receive buffers to BE
1863  */
1864 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed)
1865 {
1866         struct be_adapter *adapter = rxo->adapter;
1867         struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL;
1868         struct be_queue_info *rxq = &rxo->q;
1869         struct page *pagep = NULL;
1870         struct device *dev = &adapter->pdev->dev;
1871         struct be_eth_rx_d *rxd;
1872         u64 page_dmaaddr = 0, frag_dmaaddr;
1873         u32 posted, page_offset = 0, notify = 0;
1874
1875         page_info = &rxo->page_info_tbl[rxq->head];
1876         for (posted = 0; posted < frags_needed && !page_info->page; posted++) {
1877                 if (!pagep) {
1878                         pagep = be_alloc_pages(adapter->big_page_size, gfp);
1879                         if (unlikely(!pagep)) {
1880                                 rx_stats(rxo)->rx_post_fail++;
1881                                 break;
1882                         }
1883                         page_dmaaddr = dma_map_page(dev, pagep, 0,
1884                                                     adapter->big_page_size,
1885                                                     DMA_FROM_DEVICE);
1886                         if (dma_mapping_error(dev, page_dmaaddr)) {
1887                                 put_page(pagep);
1888                                 pagep = NULL;
1889                                 adapter->drv_stats.dma_map_errors++;
1890                                 break;
1891                         }
1892                         page_offset = 0;
1893                 } else {
1894                         get_page(pagep);
1895                         page_offset += rx_frag_size;
1896                 }
1897                 page_info->page_offset = page_offset;
1898                 page_info->page = pagep;
1899
1900                 rxd = queue_head_node(rxq);
1901                 frag_dmaaddr = page_dmaaddr + page_info->page_offset;
1902                 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF);
1903                 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr));
1904
1905                 /* Any space left in the current big page for another frag? */
1906                 if ((page_offset + rx_frag_size + rx_frag_size) >
1907                                         adapter->big_page_size) {
1908                         pagep = NULL;
1909                         page_info->last_frag = true;
1910                         dma_unmap_addr_set(page_info, bus, page_dmaaddr);
1911                 } else {
1912                         dma_unmap_addr_set(page_info, bus, frag_dmaaddr);
1913                 }
1914
1915                 prev_page_info = page_info;
1916                 queue_head_inc(rxq);
1917                 page_info = &rxo->page_info_tbl[rxq->head];
1918         }
1919
1920         /* Mark the last frag of a page when we break out of the above loop
1921          * with no more slots available in the RXQ
1922          */
1923         if (pagep) {
1924                 prev_page_info->last_frag = true;
1925                 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr);
1926         }
1927
1928         if (posted) {
1929                 atomic_add(posted, &rxq->used);
1930                 if (rxo->rx_post_starved)
1931                         rxo->rx_post_starved = false;
1932                 do {
1933                         notify = min(256u, posted);
1934                         be_rxq_notify(adapter, rxq->id, notify);
1935                         posted -= notify;
1936                 } while (posted);
1937         } else if (atomic_read(&rxq->used) == 0) {
1938                 /* Let be_worker replenish when memory is available */
1939                 rxo->rx_post_starved = true;
1940         }
1941 }
1942
1943 static struct be_eth_tx_compl *be_tx_compl_get(struct be_queue_info *tx_cq)
1944 {
1945         struct be_eth_tx_compl *txcp = queue_tail_node(tx_cq);
1946
1947         if (txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0)
1948                 return NULL;
1949
1950         rmb();
1951         be_dws_le_to_cpu(txcp, sizeof(*txcp));
1952
1953         txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0;
1954
1955         queue_tail_inc(tx_cq);
1956         return txcp;
1957 }
1958
1959 static u16 be_tx_compl_process(struct be_adapter *adapter,
1960                                struct be_tx_obj *txo, u16 last_index)
1961 {
1962         struct be_queue_info *txq = &txo->q;
1963         struct be_eth_wrb *wrb;
1964         struct sk_buff **sent_skbs = txo->sent_skb_list;
1965         struct sk_buff *sent_skb;
1966         u16 cur_index, num_wrbs = 1; /* account for hdr wrb */
1967         bool unmap_skb_hdr = true;
1968
1969         sent_skb = sent_skbs[txq->tail];
1970         BUG_ON(!sent_skb);
1971         sent_skbs[txq->tail] = NULL;
1972
1973         /* skip header wrb */
1974         queue_tail_inc(txq);
1975
1976         do {
1977                 cur_index = txq->tail;
1978                 wrb = queue_tail_node(txq);
1979                 unmap_tx_frag(&adapter->pdev->dev, wrb,
1980                               (unmap_skb_hdr && skb_headlen(sent_skb)));
1981                 unmap_skb_hdr = false;
1982
1983                 num_wrbs++;
1984                 queue_tail_inc(txq);
1985         } while (cur_index != last_index);
1986
1987         dev_consume_skb_any(sent_skb);
1988         return num_wrbs;
1989 }
1990
1991 /* Return the number of events in the event queue */
1992 static inline int events_get(struct be_eq_obj *eqo)
1993 {
1994         struct be_eq_entry *eqe;
1995         int num = 0;
1996
1997         do {
1998                 eqe = queue_tail_node(&eqo->q);
1999                 if (eqe->evt == 0)
2000                         break;
2001
2002                 rmb();
2003                 eqe->evt = 0;
2004                 num++;
2005                 queue_tail_inc(&eqo->q);
2006         } while (true);
2007
2008         return num;
2009 }
2010
2011 /* Leaves the EQ is disarmed state */
2012 static void be_eq_clean(struct be_eq_obj *eqo)
2013 {
2014         int num = events_get(eqo);
2015
2016         be_eq_notify(eqo->adapter, eqo->q.id, false, true, num);
2017 }
2018
2019 static void be_rx_cq_clean(struct be_rx_obj *rxo)
2020 {
2021         struct be_rx_page_info *page_info;
2022         struct be_queue_info *rxq = &rxo->q;
2023         struct be_queue_info *rx_cq = &rxo->cq;
2024         struct be_rx_compl_info *rxcp;
2025         struct be_adapter *adapter = rxo->adapter;
2026         int flush_wait = 0;
2027
2028         /* Consume pending rx completions.
2029          * Wait for the flush completion (identified by zero num_rcvd)
2030          * to arrive. Notify CQ even when there are no more CQ entries
2031          * for HW to flush partially coalesced CQ entries.
2032          * In Lancer, there is no need to wait for flush compl.
2033          */
2034         for (;;) {
2035                 rxcp = be_rx_compl_get(rxo);
2036                 if (!rxcp) {
2037                         if (lancer_chip(adapter))
2038                                 break;
2039
2040                         if (flush_wait++ > 10 || be_hw_error(adapter)) {
2041                                 dev_warn(&adapter->pdev->dev,
2042                                          "did not receive flush compl\n");
2043                                 break;
2044                         }
2045                         be_cq_notify(adapter, rx_cq->id, true, 0);
2046                         mdelay(1);
2047                 } else {
2048                         be_rx_compl_discard(rxo, rxcp);
2049                         be_cq_notify(adapter, rx_cq->id, false, 1);
2050                         if (rxcp->num_rcvd == 0)
2051                                 break;
2052                 }
2053         }
2054
2055         /* After cleanup, leave the CQ in unarmed state */
2056         be_cq_notify(adapter, rx_cq->id, false, 0);
2057
2058         /* Then free posted rx buffers that were not used */
2059         while (atomic_read(&rxq->used) > 0) {
2060                 page_info = get_rx_page_info(rxo);
2061                 put_page(page_info->page);
2062                 memset(page_info, 0, sizeof(*page_info));
2063         }
2064         BUG_ON(atomic_read(&rxq->used));
2065         rxq->tail = 0;
2066         rxq->head = 0;
2067 }
2068
2069 static void be_tx_compl_clean(struct be_adapter *adapter)
2070 {
2071         struct be_tx_obj *txo;
2072         struct be_queue_info *txq;
2073         struct be_eth_tx_compl *txcp;
2074         u16 end_idx, cmpl = 0, timeo = 0, num_wrbs = 0;
2075         struct sk_buff *sent_skb;
2076         bool dummy_wrb;
2077         int i, pending_txqs;
2078
2079         /* Stop polling for compls when HW has been silent for 10ms */
2080         do {
2081                 pending_txqs = adapter->num_tx_qs;
2082
2083                 for_all_tx_queues(adapter, txo, i) {
2084                         cmpl = 0;
2085                         num_wrbs = 0;
2086                         txq = &txo->q;
2087                         while ((txcp = be_tx_compl_get(&txo->cq))) {
2088                                 end_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2089                                 num_wrbs += be_tx_compl_process(adapter, txo,
2090                                                                 end_idx);
2091                                 cmpl++;
2092                         }
2093                         if (cmpl) {
2094                                 be_cq_notify(adapter, txo->cq.id, false, cmpl);
2095                                 atomic_sub(num_wrbs, &txq->used);
2096                                 timeo = 0;
2097                         }
2098                         if (atomic_read(&txq->used) == 0)
2099                                 pending_txqs--;
2100                 }
2101
2102                 if (pending_txqs == 0 || ++timeo > 10 || be_hw_error(adapter))
2103                         break;
2104
2105                 mdelay(1);
2106         } while (true);
2107
2108         for_all_tx_queues(adapter, txo, i) {
2109                 txq = &txo->q;
2110                 if (atomic_read(&txq->used))
2111                         dev_err(&adapter->pdev->dev, "%d pending tx-compls\n",
2112                                 atomic_read(&txq->used));
2113
2114                 /* free posted tx for which compls will never arrive */
2115                 while (atomic_read(&txq->used)) {
2116                         sent_skb = txo->sent_skb_list[txq->tail];
2117                         end_idx = txq->tail;
2118                         num_wrbs = wrb_cnt_for_skb(adapter, sent_skb,
2119                                                    &dummy_wrb);
2120                         index_adv(&end_idx, num_wrbs - 1, txq->len);
2121                         num_wrbs = be_tx_compl_process(adapter, txo, end_idx);
2122                         atomic_sub(num_wrbs, &txq->used);
2123                 }
2124         }
2125 }
2126
2127 static void be_evt_queues_destroy(struct be_adapter *adapter)
2128 {
2129         struct be_eq_obj *eqo;
2130         int i;
2131
2132         for_all_evt_queues(adapter, eqo, i) {
2133                 if (eqo->q.created) {
2134                         be_eq_clean(eqo);
2135                         be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ);
2136                         napi_hash_del(&eqo->napi);
2137                         netif_napi_del(&eqo->napi);
2138                 }
2139                 be_queue_free(adapter, &eqo->q);
2140         }
2141 }
2142
2143 static int be_evt_queues_create(struct be_adapter *adapter)
2144 {
2145         struct be_queue_info *eq;
2146         struct be_eq_obj *eqo;
2147         struct be_aic_obj *aic;
2148         int i, rc;
2149
2150         adapter->num_evt_qs = min_t(u16, num_irqs(adapter),
2151                                     adapter->cfg_num_qs);
2152
2153         for_all_evt_queues(adapter, eqo, i) {
2154                 netif_napi_add(adapter->netdev, &eqo->napi, be_poll,
2155                                BE_NAPI_WEIGHT);
2156                 napi_hash_add(&eqo->napi);
2157                 aic = &adapter->aic_obj[i];
2158                 eqo->adapter = adapter;
2159                 eqo->idx = i;
2160                 aic->max_eqd = BE_MAX_EQD;
2161                 aic->enable = true;
2162
2163                 eq = &eqo->q;
2164                 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN,
2165                                     sizeof(struct be_eq_entry));
2166                 if (rc)
2167                         return rc;
2168
2169                 rc = be_cmd_eq_create(adapter, eqo);
2170                 if (rc)
2171                         return rc;
2172         }
2173         return 0;
2174 }
2175
2176 static void be_mcc_queues_destroy(struct be_adapter *adapter)
2177 {
2178         struct be_queue_info *q;
2179
2180         q = &adapter->mcc_obj.q;
2181         if (q->created)
2182                 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ);
2183         be_queue_free(adapter, q);
2184
2185         q = &adapter->mcc_obj.cq;
2186         if (q->created)
2187                 be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2188         be_queue_free(adapter, q);
2189 }
2190
2191 /* Must be called only after TX qs are created as MCC shares TX EQ */
2192 static int be_mcc_queues_create(struct be_adapter *adapter)
2193 {
2194         struct be_queue_info *q, *cq;
2195
2196         cq = &adapter->mcc_obj.cq;
2197         if (be_queue_alloc(adapter, cq, MCC_CQ_LEN,
2198                            sizeof(struct be_mcc_compl)))
2199                 goto err;
2200
2201         /* Use the default EQ for MCC completions */
2202         if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0))
2203                 goto mcc_cq_free;
2204
2205         q = &adapter->mcc_obj.q;
2206         if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb)))
2207                 goto mcc_cq_destroy;
2208
2209         if (be_cmd_mccq_create(adapter, q, cq))
2210                 goto mcc_q_free;
2211
2212         return 0;
2213
2214 mcc_q_free:
2215         be_queue_free(adapter, q);
2216 mcc_cq_destroy:
2217         be_cmd_q_destroy(adapter, cq, QTYPE_CQ);
2218 mcc_cq_free:
2219         be_queue_free(adapter, cq);
2220 err:
2221         return -1;
2222 }
2223
2224 static void be_tx_queues_destroy(struct be_adapter *adapter)
2225 {
2226         struct be_queue_info *q;
2227         struct be_tx_obj *txo;
2228         u8 i;
2229
2230         for_all_tx_queues(adapter, txo, i) {
2231                 q = &txo->q;
2232                 if (q->created)
2233                         be_cmd_q_destroy(adapter, q, QTYPE_TXQ);
2234                 be_queue_free(adapter, q);
2235
2236                 q = &txo->cq;
2237                 if (q->created)
2238                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2239                 be_queue_free(adapter, q);
2240         }
2241 }
2242
2243 static int be_tx_qs_create(struct be_adapter *adapter)
2244 {
2245         struct be_queue_info *cq, *eq;
2246         struct be_tx_obj *txo;
2247         int status, i;
2248
2249         adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter));
2250
2251         for_all_tx_queues(adapter, txo, i) {
2252                 cq = &txo->cq;
2253                 status = be_queue_alloc(adapter, cq, TX_CQ_LEN,
2254                                         sizeof(struct be_eth_tx_compl));
2255                 if (status)
2256                         return status;
2257
2258                 u64_stats_init(&txo->stats.sync);
2259                 u64_stats_init(&txo->stats.sync_compl);
2260
2261                 /* If num_evt_qs is less than num_tx_qs, then more than
2262                  * one txq share an eq
2263                  */
2264                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2265                 status = be_cmd_cq_create(adapter, cq, eq, false, 3);
2266                 if (status)
2267                         return status;
2268
2269                 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN,
2270                                         sizeof(struct be_eth_wrb));
2271                 if (status)
2272                         return status;
2273
2274                 status = be_cmd_txq_create(adapter, txo);
2275                 if (status)
2276                         return status;
2277         }
2278
2279         dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n",
2280                  adapter->num_tx_qs);
2281         return 0;
2282 }
2283
2284 static void be_rx_cqs_destroy(struct be_adapter *adapter)
2285 {
2286         struct be_queue_info *q;
2287         struct be_rx_obj *rxo;
2288         int i;
2289
2290         for_all_rx_queues(adapter, rxo, i) {
2291                 q = &rxo->cq;
2292                 if (q->created)
2293                         be_cmd_q_destroy(adapter, q, QTYPE_CQ);
2294                 be_queue_free(adapter, q);
2295         }
2296 }
2297
2298 static int be_rx_cqs_create(struct be_adapter *adapter)
2299 {
2300         struct be_queue_info *eq, *cq;
2301         struct be_rx_obj *rxo;
2302         int rc, i;
2303
2304         /* We can create as many RSS rings as there are EQs. */
2305         adapter->num_rx_qs = adapter->num_evt_qs;
2306
2307         /* We'll use RSS only if atleast 2 RSS rings are supported.
2308          * When RSS is used, we'll need a default RXQ for non-IP traffic.
2309          */
2310         if (adapter->num_rx_qs > 1)
2311                 adapter->num_rx_qs++;
2312
2313         adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE;
2314         for_all_rx_queues(adapter, rxo, i) {
2315                 rxo->adapter = adapter;
2316                 cq = &rxo->cq;
2317                 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN,
2318                                     sizeof(struct be_eth_rx_compl));
2319                 if (rc)
2320                         return rc;
2321
2322                 u64_stats_init(&rxo->stats.sync);
2323                 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q;
2324                 rc = be_cmd_cq_create(adapter, cq, eq, false, 3);
2325                 if (rc)
2326                         return rc;
2327         }
2328
2329         dev_info(&adapter->pdev->dev,
2330                  "created %d RSS queue(s) and 1 default RX queue\n",
2331                  adapter->num_rx_qs - 1);
2332         return 0;
2333 }
2334
2335 static irqreturn_t be_intx(int irq, void *dev)
2336 {
2337         struct be_eq_obj *eqo = dev;
2338         struct be_adapter *adapter = eqo->adapter;
2339         int num_evts = 0;
2340
2341         /* IRQ is not expected when NAPI is scheduled as the EQ
2342          * will not be armed.
2343          * But, this can happen on Lancer INTx where it takes
2344          * a while to de-assert INTx or in BE2 where occasionaly
2345          * an interrupt may be raised even when EQ is unarmed.
2346          * If NAPI is already scheduled, then counting & notifying
2347          * events will orphan them.
2348          */
2349         if (napi_schedule_prep(&eqo->napi)) {
2350                 num_evts = events_get(eqo);
2351                 __napi_schedule(&eqo->napi);
2352                 if (num_evts)
2353                         eqo->spurious_intr = 0;
2354         }
2355         be_eq_notify(adapter, eqo->q.id, false, true, num_evts);
2356
2357         /* Return IRQ_HANDLED only for the the first spurious intr
2358          * after a valid intr to stop the kernel from branding
2359          * this irq as a bad one!
2360          */
2361         if (num_evts || eqo->spurious_intr++ == 0)
2362                 return IRQ_HANDLED;
2363         else
2364                 return IRQ_NONE;
2365 }
2366
2367 static irqreturn_t be_msix(int irq, void *dev)
2368 {
2369         struct be_eq_obj *eqo = dev;
2370
2371         be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
2372         napi_schedule(&eqo->napi);
2373         return IRQ_HANDLED;
2374 }
2375
2376 static inline bool do_gro(struct be_rx_compl_info *rxcp)
2377 {
2378         return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false;
2379 }
2380
2381 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi,
2382                          int budget, int polling)
2383 {
2384         struct be_adapter *adapter = rxo->adapter;
2385         struct be_queue_info *rx_cq = &rxo->cq;
2386         struct be_rx_compl_info *rxcp;
2387         u32 work_done;
2388         u32 frags_consumed = 0;
2389
2390         for (work_done = 0; work_done < budget; work_done++) {
2391                 rxcp = be_rx_compl_get(rxo);
2392                 if (!rxcp)
2393                         break;
2394
2395                 /* Is it a flush compl that has no data */
2396                 if (unlikely(rxcp->num_rcvd == 0))
2397                         goto loop_continue;
2398
2399                 /* Discard compl with partial DMA Lancer B0 */
2400                 if (unlikely(!rxcp->pkt_size)) {
2401                         be_rx_compl_discard(rxo, rxcp);
2402                         goto loop_continue;
2403                 }
2404
2405                 /* On BE drop pkts that arrive due to imperfect filtering in
2406                  * promiscuous mode on some skews
2407                  */
2408                 if (unlikely(rxcp->port != adapter->port_num &&
2409                              !lancer_chip(adapter))) {
2410                         be_rx_compl_discard(rxo, rxcp);
2411                         goto loop_continue;
2412                 }
2413
2414                 /* Don't do gro when we're busy_polling */
2415                 if (do_gro(rxcp) && polling != BUSY_POLLING)
2416                         be_rx_compl_process_gro(rxo, napi, rxcp);
2417                 else
2418                         be_rx_compl_process(rxo, napi, rxcp);
2419
2420 loop_continue:
2421                 frags_consumed += rxcp->num_rcvd;
2422                 be_rx_stats_update(rxo, rxcp);
2423         }
2424
2425         if (work_done) {
2426                 be_cq_notify(adapter, rx_cq->id, true, work_done);
2427
2428                 /* When an rx-obj gets into post_starved state, just
2429                  * let be_worker do the posting.
2430                  */
2431                 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM &&
2432                     !rxo->rx_post_starved)
2433                         be_post_rx_frags(rxo, GFP_ATOMIC,
2434                                          max_t(u32, MAX_RX_POST,
2435                                                frags_consumed));
2436         }
2437
2438         return work_done;
2439 }
2440
2441 static inline void be_update_tx_err(struct be_tx_obj *txo, u32 status)
2442 {
2443         switch (status) {
2444         case BE_TX_COMP_HDR_PARSE_ERR:
2445                 tx_stats(txo)->tx_hdr_parse_err++;
2446                 break;
2447         case BE_TX_COMP_NDMA_ERR:
2448                 tx_stats(txo)->tx_dma_err++;
2449                 break;
2450         case BE_TX_COMP_ACL_ERR:
2451                 tx_stats(txo)->tx_spoof_check_err++;
2452                 break;
2453         }
2454 }
2455
2456 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u32 status)
2457 {
2458         switch (status) {
2459         case LANCER_TX_COMP_LSO_ERR:
2460                 tx_stats(txo)->tx_tso_err++;
2461                 break;
2462         case LANCER_TX_COMP_HSW_DROP_MAC_ERR:
2463         case LANCER_TX_COMP_HSW_DROP_VLAN_ERR:
2464                 tx_stats(txo)->tx_spoof_check_err++;
2465                 break;
2466         case LANCER_TX_COMP_QINQ_ERR:
2467                 tx_stats(txo)->tx_qinq_err++;
2468                 break;
2469         case LANCER_TX_COMP_PARITY_ERR:
2470                 tx_stats(txo)->tx_internal_parity_err++;
2471                 break;
2472         case LANCER_TX_COMP_DMA_ERR:
2473                 tx_stats(txo)->tx_dma_err++;
2474                 break;
2475         }
2476 }
2477
2478 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo,
2479                           int idx)
2480 {
2481         struct be_eth_tx_compl *txcp;
2482         int num_wrbs = 0, work_done = 0;
2483         u32 compl_status;
2484         u16 last_idx;
2485
2486         while ((txcp = be_tx_compl_get(&txo->cq))) {
2487                 last_idx = GET_TX_COMPL_BITS(wrb_index, txcp);
2488                 num_wrbs += be_tx_compl_process(adapter, txo, last_idx);
2489                 work_done++;
2490
2491                 compl_status = GET_TX_COMPL_BITS(status, txcp);
2492                 if (compl_status) {
2493                         if (lancer_chip(adapter))
2494                                 lancer_update_tx_err(txo, compl_status);
2495                         else
2496                                 be_update_tx_err(txo, compl_status);
2497                 }
2498         }
2499
2500         if (work_done) {
2501                 be_cq_notify(adapter, txo->cq.id, true, work_done);
2502                 atomic_sub(num_wrbs, &txo->q.used);
2503
2504                 /* As Tx wrbs have been freed up, wake up netdev queue
2505                  * if it was stopped due to lack of tx wrbs.  */
2506                 if (__netif_subqueue_stopped(adapter->netdev, idx) &&
2507                     atomic_read(&txo->q.used) < txo->q.len / 2) {
2508                         netif_wake_subqueue(adapter->netdev, idx);
2509                 }
2510
2511                 u64_stats_update_begin(&tx_stats(txo)->sync_compl);
2512                 tx_stats(txo)->tx_compl += work_done;
2513                 u64_stats_update_end(&tx_stats(txo)->sync_compl);
2514         }
2515 }
2516
2517 int be_poll(struct napi_struct *napi, int budget)
2518 {
2519         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2520         struct be_adapter *adapter = eqo->adapter;
2521         int max_work = 0, work, i, num_evts;
2522         struct be_rx_obj *rxo;
2523         struct be_tx_obj *txo;
2524
2525         num_evts = events_get(eqo);
2526
2527         for_all_tx_queues_on_eq(adapter, eqo, txo, i)
2528                 be_process_tx(adapter, txo, i);
2529
2530         if (be_lock_napi(eqo)) {
2531                 /* This loop will iterate twice for EQ0 in which
2532                  * completions of the last RXQ (default one) are also processed
2533                  * For other EQs the loop iterates only once
2534                  */
2535                 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2536                         work = be_process_rx(rxo, napi, budget, NAPI_POLLING);
2537                         max_work = max(work, max_work);
2538                 }
2539                 be_unlock_napi(eqo);
2540         } else {
2541                 max_work = budget;
2542         }
2543
2544         if (is_mcc_eqo(eqo))
2545                 be_process_mcc(adapter);
2546
2547         if (max_work < budget) {
2548                 napi_complete(napi);
2549                 be_eq_notify(adapter, eqo->q.id, true, false, num_evts);
2550         } else {
2551                 /* As we'll continue in polling mode, count and clear events */
2552                 be_eq_notify(adapter, eqo->q.id, false, false, num_evts);
2553         }
2554         return max_work;
2555 }
2556
2557 #ifdef CONFIG_NET_RX_BUSY_POLL
2558 static int be_busy_poll(struct napi_struct *napi)
2559 {
2560         struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi);
2561         struct be_adapter *adapter = eqo->adapter;
2562         struct be_rx_obj *rxo;
2563         int i, work = 0;
2564
2565         if (!be_lock_busy_poll(eqo))
2566                 return LL_FLUSH_BUSY;
2567
2568         for_all_rx_queues_on_eq(adapter, eqo, rxo, i) {
2569                 work = be_process_rx(rxo, napi, 4, BUSY_POLLING);
2570                 if (work)
2571                         break;
2572         }
2573
2574         be_unlock_busy_poll(eqo);
2575         return work;
2576 }
2577 #endif
2578
2579 void be_detect_error(struct be_adapter *adapter)
2580 {
2581         u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0;
2582         u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2583         u32 i;
2584         bool error_detected = false;
2585         struct device *dev = &adapter->pdev->dev;
2586         struct net_device *netdev = adapter->netdev;
2587
2588         if (be_hw_error(adapter))
2589                 return;
2590
2591         if (lancer_chip(adapter)) {
2592                 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET);
2593                 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2594                         sliport_err1 = ioread32(adapter->db +
2595                                                 SLIPORT_ERROR1_OFFSET);
2596                         sliport_err2 = ioread32(adapter->db +
2597                                                 SLIPORT_ERROR2_OFFSET);
2598                         adapter->hw_error = true;
2599                         /* Do not log error messages if its a FW reset */
2600                         if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 &&
2601                             sliport_err2 == SLIPORT_ERROR_FW_RESET2) {
2602                                 dev_info(dev, "Firmware update in progress\n");
2603                         } else {
2604                                 error_detected = true;
2605                                 dev_err(dev, "Error detected in the card\n");
2606                                 dev_err(dev, "ERR: sliport status 0x%x\n",
2607                                         sliport_status);
2608                                 dev_err(dev, "ERR: sliport error1 0x%x\n",
2609                                         sliport_err1);
2610                                 dev_err(dev, "ERR: sliport error2 0x%x\n",
2611                                         sliport_err2);
2612                         }
2613                 }
2614         } else {
2615                 pci_read_config_dword(adapter->pdev,
2616                                       PCICFG_UE_STATUS_LOW, &ue_lo);
2617                 pci_read_config_dword(adapter->pdev,
2618                                       PCICFG_UE_STATUS_HIGH, &ue_hi);
2619                 pci_read_config_dword(adapter->pdev,
2620                                       PCICFG_UE_STATUS_LOW_MASK, &ue_lo_mask);
2621                 pci_read_config_dword(adapter->pdev,
2622                                       PCICFG_UE_STATUS_HI_MASK, &ue_hi_mask);
2623
2624                 ue_lo = (ue_lo & ~ue_lo_mask);
2625                 ue_hi = (ue_hi & ~ue_hi_mask);
2626
2627                 /* On certain platforms BE hardware can indicate spurious UEs.
2628                  * Allow HW to stop working completely in case of a real UE.
2629                  * Hence not setting the hw_error for UE detection.
2630                  */
2631
2632                 if (ue_lo || ue_hi) {
2633                         error_detected = true;
2634                         dev_err(dev,
2635                                 "Unrecoverable Error detected in the adapter");
2636                         dev_err(dev, "Please reboot server to recover");
2637                         if (skyhawk_chip(adapter))
2638                                 adapter->hw_error = true;
2639                         for (i = 0; ue_lo; ue_lo >>= 1, i++) {
2640                                 if (ue_lo & 1)
2641                                         dev_err(dev, "UE: %s bit set\n",
2642                                                 ue_status_low_desc[i]);
2643                         }
2644                         for (i = 0; ue_hi; ue_hi >>= 1, i++) {
2645                                 if (ue_hi & 1)
2646                                         dev_err(dev, "UE: %s bit set\n",
2647                                                 ue_status_hi_desc[i]);
2648                         }
2649                 }
2650         }
2651         if (error_detected)
2652                 netif_carrier_off(netdev);
2653 }
2654
2655 static void be_msix_disable(struct be_adapter *adapter)
2656 {
2657         if (msix_enabled(adapter)) {
2658                 pci_disable_msix(adapter->pdev);
2659                 adapter->num_msix_vec = 0;
2660                 adapter->num_msix_roce_vec = 0;
2661         }
2662 }
2663
2664 static int be_msix_enable(struct be_adapter *adapter)
2665 {
2666         int i, num_vec;
2667         struct device *dev = &adapter->pdev->dev;
2668
2669         /* If RoCE is supported, program the max number of NIC vectors that
2670          * may be configured via set-channels, along with vectors needed for
2671          * RoCe. Else, just program the number we'll use initially.
2672          */
2673         if (be_roce_supported(adapter))
2674                 num_vec = min_t(int, 2 * be_max_eqs(adapter),
2675                                 2 * num_online_cpus());
2676         else
2677                 num_vec = adapter->cfg_num_qs;
2678
2679         for (i = 0; i < num_vec; i++)
2680                 adapter->msix_entries[i].entry = i;
2681
2682         num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
2683                                         MIN_MSIX_VECTORS, num_vec);
2684         if (num_vec < 0)
2685                 goto fail;
2686
2687         if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) {
2688                 adapter->num_msix_roce_vec = num_vec / 2;
2689                 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n",
2690                          adapter->num_msix_roce_vec);
2691         }
2692
2693         adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec;
2694
2695         dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n",
2696                  adapter->num_msix_vec);
2697         return 0;
2698
2699 fail:
2700         dev_warn(dev, "MSIx enable failed\n");
2701
2702         /* INTx is not supported in VFs, so fail probe if enable_msix fails */
2703         if (!be_physfn(adapter))
2704                 return num_vec;
2705         return 0;
2706 }
2707
2708 static inline int be_msix_vec_get(struct be_adapter *adapter,
2709                                   struct be_eq_obj *eqo)
2710 {
2711         return adapter->msix_entries[eqo->msix_idx].vector;
2712 }
2713
2714 static int be_msix_register(struct be_adapter *adapter)
2715 {
2716         struct net_device *netdev = adapter->netdev;
2717         struct be_eq_obj *eqo;
2718         int status, i, vec;
2719
2720         for_all_evt_queues(adapter, eqo, i) {
2721                 sprintf(eqo->desc, "%s-q%d", netdev->name, i);
2722                 vec = be_msix_vec_get(adapter, eqo);
2723                 status = request_irq(vec, be_msix, 0, eqo->desc, eqo);
2724                 if (status)
2725                         goto err_msix;
2726         }
2727
2728         return 0;
2729 err_msix:
2730         for (i--, eqo = &adapter->eq_obj[i]; i >= 0; i--, eqo--)
2731                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2732         dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n",
2733                  status);
2734         be_msix_disable(adapter);
2735         return status;
2736 }
2737
2738 static int be_irq_register(struct be_adapter *adapter)
2739 {
2740         struct net_device *netdev = adapter->netdev;
2741         int status;
2742
2743         if (msix_enabled(adapter)) {
2744                 status = be_msix_register(adapter);
2745                 if (status == 0)
2746                         goto done;
2747                 /* INTx is not supported for VF */
2748                 if (!be_physfn(adapter))
2749                         return status;
2750         }
2751
2752         /* INTx: only the first EQ is used */
2753         netdev->irq = adapter->pdev->irq;
2754         status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name,
2755                              &adapter->eq_obj[0]);
2756         if (status) {
2757                 dev_err(&adapter->pdev->dev,
2758                         "INTx request IRQ failed - err %d\n", status);
2759                 return status;
2760         }
2761 done:
2762         adapter->isr_registered = true;
2763         return 0;
2764 }
2765
2766 static void be_irq_unregister(struct be_adapter *adapter)
2767 {
2768         struct net_device *netdev = adapter->netdev;
2769         struct be_eq_obj *eqo;
2770         int i;
2771
2772         if (!adapter->isr_registered)
2773                 return;
2774
2775         /* INTx */
2776         if (!msix_enabled(adapter)) {
2777                 free_irq(netdev->irq, &adapter->eq_obj[0]);
2778                 goto done;
2779         }
2780
2781         /* MSIx */
2782         for_all_evt_queues(adapter, eqo, i)
2783                 free_irq(be_msix_vec_get(adapter, eqo), eqo);
2784
2785 done:
2786         adapter->isr_registered = false;
2787 }
2788
2789 static void be_rx_qs_destroy(struct be_adapter *adapter)
2790 {
2791         struct be_queue_info *q;
2792         struct be_rx_obj *rxo;
2793         int i;
2794
2795         for_all_rx_queues(adapter, rxo, i) {
2796                 q = &rxo->q;
2797                 if (q->created) {
2798                         be_cmd_rxq_destroy(adapter, q);
2799                         be_rx_cq_clean(rxo);
2800                 }
2801                 be_queue_free(adapter, q);
2802         }
2803 }
2804
2805 static int be_close(struct net_device *netdev)
2806 {
2807         struct be_adapter *adapter = netdev_priv(netdev);
2808         struct be_eq_obj *eqo;
2809         int i;
2810
2811         /* This protection is needed as be_close() may be called even when the
2812          * adapter is in cleared state (after eeh perm failure)
2813          */
2814         if (!(adapter->flags & BE_FLAGS_SETUP_DONE))
2815                 return 0;
2816
2817         be_roce_dev_close(adapter);
2818
2819         if (adapter->flags & BE_FLAGS_NAPI_ENABLED) {
2820                 for_all_evt_queues(adapter, eqo, i) {
2821                         napi_disable(&eqo->napi);
2822                         be_disable_busy_poll(eqo);
2823                 }
2824                 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED;
2825         }
2826
2827         be_async_mcc_disable(adapter);
2828
2829         /* Wait for all pending tx completions to arrive so that
2830          * all tx skbs are freed.
2831          */
2832         netif_tx_disable(netdev);
2833         be_tx_compl_clean(adapter);
2834
2835         be_rx_qs_destroy(adapter);
2836
2837         for (i = 1; i < (adapter->uc_macs + 1); i++)
2838                 be_cmd_pmac_del(adapter, adapter->if_handle,
2839                                 adapter->pmac_id[i], 0);
2840         adapter->uc_macs = 0;
2841
2842         for_all_evt_queues(adapter, eqo, i) {
2843                 if (msix_enabled(adapter))
2844                         synchronize_irq(be_msix_vec_get(adapter, eqo));
2845                 else
2846                         synchronize_irq(netdev->irq);
2847                 be_eq_clean(eqo);
2848         }
2849
2850         be_irq_unregister(adapter);
2851
2852         return 0;
2853 }
2854
2855 static int be_rx_qs_create(struct be_adapter *adapter)
2856 {
2857         struct rss_info *rss = &adapter->rss_info;
2858         u8 rss_key[RSS_HASH_KEY_LEN];
2859         struct be_rx_obj *rxo;
2860         int rc, i, j;
2861
2862         for_all_rx_queues(adapter, rxo, i) {
2863                 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN,
2864                                     sizeof(struct be_eth_rx_d));
2865                 if (rc)
2866                         return rc;
2867         }
2868
2869         /* The FW would like the default RXQ to be created first */
2870         rxo = default_rxo(adapter);
2871         rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, rx_frag_size,
2872                                adapter->if_handle, false, &rxo->rss_id);
2873         if (rc)
2874                 return rc;
2875
2876         for_all_rss_queues(adapter, rxo, i) {
2877                 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id,
2878                                        rx_frag_size, adapter->if_handle,
2879                                        true, &rxo->rss_id);
2880                 if (rc)
2881                         return rc;
2882         }
2883
2884         if (be_multi_rxq(adapter)) {
2885                 for (j = 0; j < RSS_INDIR_TABLE_LEN;
2886                         j += adapter->num_rx_qs - 1) {
2887                         for_all_rss_queues(adapter, rxo, i) {
2888                                 if ((j + i) >= RSS_INDIR_TABLE_LEN)
2889                                         break;
2890                                 rss->rsstable[j + i] = rxo->rss_id;
2891                                 rss->rss_queue[j + i] = i;
2892                         }
2893                 }
2894                 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 |
2895                         RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6;
2896
2897                 if (!BEx_chip(adapter))
2898                         rss->rss_flags |= RSS_ENABLE_UDP_IPV4 |
2899                                 RSS_ENABLE_UDP_IPV6;
2900         } else {
2901                 /* Disable RSS, if only default RX Q is created */
2902                 rss->rss_flags = RSS_ENABLE_NONE;
2903         }
2904
2905         netdev_rss_key_fill(rss_key, RSS_HASH_KEY_LEN);
2906         rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags,
2907                                128, rss_key);
2908         if (rc) {
2909                 rss->rss_flags = RSS_ENABLE_NONE;
2910                 return rc;
2911         }
2912
2913         memcpy(rss->rss_hkey, rss_key, RSS_HASH_KEY_LEN);
2914
2915         /* First time posting */
2916         for_all_rx_queues(adapter, rxo, i)
2917                 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
2918         return 0;
2919 }
2920
2921 static int be_open(struct net_device *netdev)
2922 {
2923         struct be_adapter *adapter = netdev_priv(netdev);
2924         struct be_eq_obj *eqo;
2925         struct be_rx_obj *rxo;
2926         struct be_tx_obj *txo;
2927         u8 link_status;
2928         int status, i;
2929
2930         status = be_rx_qs_create(adapter);
2931         if (status)
2932                 goto err;
2933
2934         status = be_irq_register(adapter);
2935         if (status)
2936                 goto err;
2937
2938         for_all_rx_queues(adapter, rxo, i)
2939                 be_cq_notify(adapter, rxo->cq.id, true, 0);
2940
2941         for_all_tx_queues(adapter, txo, i)
2942                 be_cq_notify(adapter, txo->cq.id, true, 0);
2943
2944         be_async_mcc_enable(adapter);
2945
2946         for_all_evt_queues(adapter, eqo, i) {
2947                 napi_enable(&eqo->napi);
2948                 be_enable_busy_poll(eqo);
2949                 be_eq_notify(adapter, eqo->q.id, true, true, 0);
2950         }
2951         adapter->flags |= BE_FLAGS_NAPI_ENABLED;
2952
2953         status = be_cmd_link_status_query(adapter, NULL, &link_status, 0);
2954         if (!status)
2955                 be_link_status_update(adapter, link_status);
2956
2957         netif_tx_start_all_queues(netdev);
2958         be_roce_dev_open(adapter);
2959
2960 #ifdef CONFIG_BE2NET_VXLAN
2961         if (skyhawk_chip(adapter))
2962                 vxlan_get_rx_port(netdev);
2963 #endif
2964
2965         return 0;
2966 err:
2967         be_close(adapter->netdev);
2968         return -EIO;
2969 }
2970
2971 static int be_setup_wol(struct be_adapter *adapter, bool enable)
2972 {
2973         struct be_dma_mem cmd;
2974         int status = 0;
2975         u8 mac[ETH_ALEN];
2976
2977         memset(mac, 0, ETH_ALEN);
2978
2979         cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config);
2980         cmd.va = dma_zalloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma,
2981                                      GFP_KERNEL);
2982         if (!cmd.va)
2983                 return -ENOMEM;
2984
2985         if (enable) {
2986                 status = pci_write_config_dword(adapter->pdev,
2987                                                 PCICFG_PM_CONTROL_OFFSET,
2988                                                 PCICFG_PM_CONTROL_MASK);
2989                 if (status) {
2990                         dev_err(&adapter->pdev->dev,
2991                                 "Could not enable Wake-on-lan\n");
2992                         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va,
2993                                           cmd.dma);
2994                         return status;
2995                 }
2996                 status = be_cmd_enable_magic_wol(adapter,
2997                                                  adapter->netdev->dev_addr,
2998                                                  &cmd);
2999                 pci_enable_wake(adapter->pdev, PCI_D3hot, 1);
3000                 pci_enable_wake(adapter->pdev, PCI_D3cold, 1);
3001         } else {
3002                 status = be_cmd_enable_magic_wol(adapter, mac, &cmd);
3003                 pci_enable_wake(adapter->pdev, PCI_D3hot, 0);
3004                 pci_enable_wake(adapter->pdev, PCI_D3cold, 0);
3005         }
3006
3007         dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma);
3008         return status;
3009 }
3010
3011 /*
3012  * Generate a seed MAC address from the PF MAC Address using jhash.
3013  * MAC Address for VFs are assigned incrementally starting from the seed.
3014  * These addresses are programmed in the ASIC by the PF and the VF driver
3015  * queries for the MAC address during its probe.
3016  */
3017 static int be_vf_eth_addr_config(struct be_adapter *adapter)
3018 {
3019         u32 vf;
3020         int status = 0;
3021         u8 mac[ETH_ALEN];
3022         struct be_vf_cfg *vf_cfg;
3023
3024         be_vf_eth_addr_generate(adapter, mac);
3025
3026         for_all_vfs(adapter, vf_cfg, vf) {
3027                 if (BEx_chip(adapter))
3028                         status = be_cmd_pmac_add(adapter, mac,
3029                                                  vf_cfg->if_handle,
3030                                                  &vf_cfg->pmac_id, vf + 1);
3031                 else
3032                         status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle,
3033                                                 vf + 1);
3034
3035                 if (status)
3036                         dev_err(&adapter->pdev->dev,
3037                                 "Mac address assignment failed for VF %d\n",
3038                                 vf);
3039                 else
3040                         memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3041
3042                 mac[5] += 1;
3043         }
3044         return status;
3045 }
3046
3047 static int be_vfs_mac_query(struct be_adapter *adapter)
3048 {
3049         int status, vf;
3050         u8 mac[ETH_ALEN];
3051         struct be_vf_cfg *vf_cfg;
3052
3053         for_all_vfs(adapter, vf_cfg, vf) {
3054                 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id,
3055                                                mac, vf_cfg->if_handle,
3056                                                false, vf+1);
3057                 if (status)
3058                         return status;
3059                 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN);
3060         }
3061         return 0;
3062 }
3063
3064 static void be_vf_clear(struct be_adapter *adapter)
3065 {
3066         struct be_vf_cfg *vf_cfg;
3067         u32 vf;
3068
3069         if (pci_vfs_assigned(adapter->pdev)) {
3070                 dev_warn(&adapter->pdev->dev,
3071                          "VFs are assigned to VMs: not disabling VFs\n");
3072                 goto done;
3073         }
3074
3075         pci_disable_sriov(adapter->pdev);
3076
3077         for_all_vfs(adapter, vf_cfg, vf) {
3078                 if (BEx_chip(adapter))
3079                         be_cmd_pmac_del(adapter, vf_cfg->if_handle,
3080                                         vf_cfg->pmac_id, vf + 1);
3081                 else
3082                         be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle,
3083                                        vf + 1);
3084
3085                 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1);
3086         }
3087 done:
3088         kfree(adapter->vf_cfg);
3089         adapter->num_vfs = 0;
3090         adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED;
3091 }
3092
3093 static void be_clear_queues(struct be_adapter *adapter)
3094 {
3095         be_mcc_queues_destroy(adapter);
3096         be_rx_cqs_destroy(adapter);
3097         be_tx_queues_destroy(adapter);
3098         be_evt_queues_destroy(adapter);
3099 }
3100
3101 static void be_cancel_worker(struct be_adapter *adapter)
3102 {
3103         if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) {
3104                 cancel_delayed_work_sync(&adapter->work);
3105                 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED;
3106         }
3107 }
3108
3109 static void be_mac_clear(struct be_adapter *adapter)
3110 {
3111         int i;
3112
3113         if (adapter->pmac_id) {
3114                 for (i = 0; i < (adapter->uc_macs + 1); i++)
3115                         be_cmd_pmac_del(adapter, adapter->if_handle,
3116                                         adapter->pmac_id[i], 0);
3117                 adapter->uc_macs = 0;
3118
3119                 kfree(adapter->pmac_id);
3120                 adapter->pmac_id = NULL;
3121         }
3122 }
3123
3124 #ifdef CONFIG_BE2NET_VXLAN
3125 static void be_disable_vxlan_offloads(struct be_adapter *adapter)
3126 {
3127         struct net_device *netdev = adapter->netdev;
3128
3129         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)
3130                 be_cmd_manage_iface(adapter, adapter->if_handle,
3131                                     OP_CONVERT_TUNNEL_TO_NORMAL);
3132
3133         if (adapter->vxlan_port)
3134                 be_cmd_set_vxlan_port(adapter, 0);
3135
3136         adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS;
3137         adapter->vxlan_port = 0;
3138
3139         netdev->hw_enc_features = 0;
3140         netdev->hw_features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3141         netdev->features &= ~(NETIF_F_GSO_UDP_TUNNEL);
3142 }
3143 #endif
3144
3145 static int be_clear(struct be_adapter *adapter)
3146 {
3147         be_cancel_worker(adapter);
3148
3149         if (sriov_enabled(adapter))
3150                 be_vf_clear(adapter);
3151
3152         /* Re-configure FW to distribute resources evenly across max-supported
3153          * number of VFs, only when VFs are not already enabled.
3154          */
3155         if (be_physfn(adapter) && !pci_vfs_assigned(adapter->pdev))
3156                 be_cmd_set_sriov_config(adapter, adapter->pool_res,
3157                                         pci_sriov_get_totalvfs(adapter->pdev));
3158
3159 #ifdef CONFIG_BE2NET_VXLAN
3160         be_disable_vxlan_offloads(adapter);
3161 #endif
3162         /* delete the primary mac along with the uc-mac list */
3163         be_mac_clear(adapter);
3164
3165         be_cmd_if_destroy(adapter, adapter->if_handle,  0);
3166
3167         be_clear_queues(adapter);
3168
3169         be_msix_disable(adapter);
3170         adapter->flags &= ~BE_FLAGS_SETUP_DONE;
3171         return 0;
3172 }
3173
3174 static int be_vfs_if_create(struct be_adapter *adapter)
3175 {
3176         struct be_resources res = {0};
3177         struct be_vf_cfg *vf_cfg;
3178         u32 cap_flags, en_flags, vf;
3179         int status = 0;
3180
3181         cap_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3182                     BE_IF_FLAGS_MULTICAST;
3183
3184         for_all_vfs(adapter, vf_cfg, vf) {
3185                 if (!BE3_chip(adapter)) {
3186                         status = be_cmd_get_profile_config(adapter, &res,
3187                                                            vf + 1);
3188                         if (!status)
3189                                 cap_flags = res.if_cap_flags;
3190                 }
3191
3192                 /* If a FW profile exists, then cap_flags are updated */
3193                 en_flags = cap_flags & (BE_IF_FLAGS_UNTAGGED |
3194                                         BE_IF_FLAGS_BROADCAST |
3195                                         BE_IF_FLAGS_MULTICAST);
3196                 status =
3197                     be_cmd_if_create(adapter, cap_flags, en_flags,
3198                                      &vf_cfg->if_handle, vf + 1);
3199                 if (status)
3200                         goto err;
3201         }
3202 err:
3203         return status;
3204 }
3205
3206 static int be_vf_setup_init(struct be_adapter *adapter)
3207 {
3208         struct be_vf_cfg *vf_cfg;
3209         int vf;
3210
3211         adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg),
3212                                   GFP_KERNEL);
3213         if (!adapter->vf_cfg)
3214                 return -ENOMEM;
3215
3216         for_all_vfs(adapter, vf_cfg, vf) {
3217                 vf_cfg->if_handle = -1;
3218                 vf_cfg->pmac_id = -1;
3219         }
3220         return 0;
3221 }
3222
3223 static int be_vf_setup(struct be_adapter *adapter)
3224 {
3225         struct device *dev = &adapter->pdev->dev;
3226         struct be_vf_cfg *vf_cfg;
3227         int status, old_vfs, vf;
3228         u32 privileges;
3229
3230         old_vfs = pci_num_vf(adapter->pdev);
3231
3232         status = be_vf_setup_init(adapter);
3233         if (status)
3234                 goto err;
3235
3236         if (old_vfs) {
3237                 for_all_vfs(adapter, vf_cfg, vf) {
3238                         status = be_cmd_get_if_id(adapter, vf_cfg, vf);
3239                         if (status)
3240                                 goto err;
3241                 }
3242
3243                 status = be_vfs_mac_query(adapter);
3244                 if (status)
3245                         goto err;
3246         } else {
3247                 status = be_vfs_if_create(adapter);
3248                 if (status)
3249                         goto err;
3250
3251                 status = be_vf_eth_addr_config(adapter);
3252                 if (status)
3253                         goto err;
3254         }
3255
3256         for_all_vfs(adapter, vf_cfg, vf) {
3257                 /* Allow VFs to programs MAC/VLAN filters */
3258                 status = be_cmd_get_fn_privileges(adapter, &privileges, vf + 1);
3259                 if (!status && !(privileges & BE_PRIV_FILTMGMT)) {
3260                         status = be_cmd_set_fn_privileges(adapter,
3261                                                           privileges |
3262                                                           BE_PRIV_FILTMGMT,
3263                                                           vf + 1);
3264                         if (!status)
3265                                 dev_info(dev, "VF%d has FILTMGMT privilege\n",
3266                                          vf);
3267                 }
3268
3269                 /* Allow full available bandwidth */
3270                 if (!old_vfs)
3271                         be_cmd_config_qos(adapter, 0, 0, vf + 1);
3272
3273                 if (!old_vfs) {
3274                         be_cmd_enable_vf(adapter, vf + 1);
3275                         be_cmd_set_logical_link_config(adapter,
3276                                                        IFLA_VF_LINK_STATE_AUTO,
3277                                                        vf+1);
3278                 }
3279         }
3280
3281         if (!old_vfs) {
3282                 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs);
3283                 if (status) {
3284                         dev_err(dev, "SRIOV enable failed\n");
3285                         adapter->num_vfs = 0;
3286                         goto err;
3287                 }
3288         }
3289
3290         adapter->flags |= BE_FLAGS_SRIOV_ENABLED;
3291         return 0;
3292 err:
3293         dev_err(dev, "VF setup failed\n");
3294         be_vf_clear(adapter);
3295         return status;
3296 }
3297
3298 /* Converting function_mode bits on BE3 to SH mc_type enums */
3299
3300 static u8 be_convert_mc_type(u32 function_mode)
3301 {
3302         if (function_mode & VNIC_MODE && function_mode & QNQ_MODE)
3303                 return vNIC1;
3304         else if (function_mode & QNQ_MODE)
3305                 return FLEX10;
3306         else if (function_mode & VNIC_MODE)
3307                 return vNIC2;
3308         else if (function_mode & UMC_ENABLED)
3309                 return UMC;
3310         else
3311                 return MC_NONE;
3312 }
3313
3314 /* On BE2/BE3 FW does not suggest the supported limits */
3315 static void BEx_get_resources(struct be_adapter *adapter,
3316                               struct be_resources *res)
3317 {
3318         bool use_sriov = adapter->num_vfs ? 1 : 0;
3319
3320         if (be_physfn(adapter))
3321                 res->max_uc_mac = BE_UC_PMAC_COUNT;
3322         else
3323                 res->max_uc_mac = BE_VF_UC_PMAC_COUNT;
3324
3325         adapter->mc_type = be_convert_mc_type(adapter->function_mode);
3326
3327         if (be_is_mc(adapter)) {
3328                 /* Assuming that there are 4 channels per port,
3329                  * when multi-channel is enabled
3330                  */
3331                 if (be_is_qnq_mode(adapter))
3332                         res->max_vlans = BE_NUM_VLANS_SUPPORTED/8;
3333                 else
3334                         /* In a non-qnq multichannel mode, the pvid
3335                          * takes up one vlan entry
3336                          */
3337                         res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1;
3338         } else {
3339                 res->max_vlans = BE_NUM_VLANS_SUPPORTED;
3340         }
3341
3342         res->max_mcast_mac = BE_MAX_MC;
3343
3344         /* 1) For BE3 1Gb ports, FW does not support multiple TXQs
3345          * 2) Create multiple TX rings on a BE3-R multi-channel interface
3346          *    *only* if it is RSS-capable.
3347          */
3348         if (BE2_chip(adapter) || use_sriov ||  (adapter->port_num > 1) ||
3349             !be_physfn(adapter) || (be_is_mc(adapter) &&
3350             !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) {
3351                 res->max_tx_qs = 1;
3352         } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) {
3353                 struct be_resources super_nic_res = {0};
3354
3355                 /* On a SuperNIC profile, the driver needs to use the
3356                  * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits
3357                  */
3358                 be_cmd_get_profile_config(adapter, &super_nic_res, 0);
3359                 /* Some old versions of BE3 FW don't report max_tx_qs value */
3360                 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS;
3361         } else {
3362                 res->max_tx_qs = BE3_MAX_TX_QS;
3363         }
3364
3365         if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) &&
3366             !use_sriov && be_physfn(adapter))
3367                 res->max_rss_qs = (adapter->be3_native) ?
3368                                            BE3_MAX_RSS_QS : BE2_MAX_RSS_QS;
3369         res->max_rx_qs = res->max_rss_qs + 1;
3370
3371         if (be_physfn(adapter))
3372                 res->max_evt_qs = (be_max_vfs(adapter) > 0) ?
3373                                         BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS;
3374         else
3375                 res->max_evt_qs = 1;
3376
3377         res->if_cap_flags = BE_IF_CAP_FLAGS_WANT;
3378         if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS))
3379                 res->if_cap_flags &= ~BE_IF_FLAGS_RSS;
3380 }
3381
3382 static void be_setup_init(struct be_adapter *adapter)
3383 {
3384         adapter->vlan_prio_bmap = 0xff;
3385         adapter->phy.link_speed = -1;
3386         adapter->if_handle = -1;
3387         adapter->be3_native = false;
3388         adapter->promiscuous = false;
3389         if (be_physfn(adapter))
3390                 adapter->cmd_privileges = MAX_PRIVILEGES;
3391         else
3392                 adapter->cmd_privileges = MIN_PRIVILEGES;
3393 }
3394
3395 static int be_get_sriov_config(struct be_adapter *adapter)
3396 {
3397         struct device *dev = &adapter->pdev->dev;
3398         struct be_resources res = {0};
3399         int max_vfs, old_vfs;
3400
3401         /* Some old versions of BE3 FW don't report max_vfs value */
3402         be_cmd_get_profile_config(adapter, &res, 0);
3403
3404         if (BE3_chip(adapter) && !res.max_vfs) {
3405                 max_vfs = pci_sriov_get_totalvfs(adapter->pdev);
3406                 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0;
3407         }
3408
3409         adapter->pool_res = res;
3410
3411         if (!be_max_vfs(adapter)) {
3412                 if (num_vfs)
3413                         dev_warn(dev, "SRIOV is disabled. Ignoring num_vfs\n");
3414                 adapter->num_vfs = 0;
3415                 return 0;
3416         }
3417
3418         pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter));
3419
3420         /* validate num_vfs module param */
3421         old_vfs = pci_num_vf(adapter->pdev);
3422         if (old_vfs) {
3423                 dev_info(dev, "%d VFs are already enabled\n", old_vfs);
3424                 if (old_vfs != num_vfs)
3425                         dev_warn(dev, "Ignoring num_vfs=%d setting\n", num_vfs);
3426                 adapter->num_vfs = old_vfs;
3427         } else {
3428                 if (num_vfs > be_max_vfs(adapter)) {
3429                         dev_info(dev, "Resources unavailable to init %d VFs\n",
3430                                  num_vfs);
3431                         dev_info(dev, "Limiting to %d VFs\n",
3432                                  be_max_vfs(adapter));
3433                 }
3434                 adapter->num_vfs = min_t(u16, num_vfs, be_max_vfs(adapter));
3435         }
3436
3437         return 0;
3438 }
3439
3440 static int be_get_resources(struct be_adapter *adapter)
3441 {
3442         struct device *dev = &adapter->pdev->dev;
3443         struct be_resources res = {0};
3444         int status;
3445
3446         if (BEx_chip(adapter)) {
3447                 BEx_get_resources(adapter, &res);
3448                 adapter->res = res;
3449         }
3450
3451         /* For Lancer, SH etc read per-function resource limits from FW.
3452          * GET_FUNC_CONFIG returns per function guaranteed limits.
3453          * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits
3454          */
3455         if (!BEx_chip(adapter)) {
3456                 status = be_cmd_get_func_config(adapter, &res);
3457                 if (status)
3458                         return status;
3459
3460                 /* If RoCE may be enabled stash away half the EQs for RoCE */
3461                 if (be_roce_supported(adapter))
3462                         res.max_evt_qs /= 2;
3463                 adapter->res = res;
3464         }
3465
3466         dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n",
3467                  be_max_txqs(adapter), be_max_rxqs(adapter),
3468                  be_max_rss(adapter), be_max_eqs(adapter),
3469                  be_max_vfs(adapter));
3470         dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n",
3471                  be_max_uc(adapter), be_max_mc(adapter),
3472                  be_max_vlans(adapter));
3473
3474         return 0;
3475 }
3476
3477 static void be_sriov_config(struct be_adapter *adapter)
3478 {
3479         struct device *dev = &adapter->pdev->dev;
3480         int status;
3481
3482         status = be_get_sriov_config(adapter);
3483         if (status) {
3484                 dev_err(dev, "Failed to query SR-IOV configuration\n");
3485                 dev_err(dev, "SR-IOV cannot be enabled\n");
3486                 return;
3487         }
3488
3489         /* When the HW is in SRIOV capable configuration, the PF-pool
3490          * resources are equally distributed across the max-number of
3491          * VFs. The user may request only a subset of the max-vfs to be
3492          * enabled. Based on num_vfs, redistribute the resources across
3493          * num_vfs so that each VF will have access to more number of
3494          * resources. This facility is not available in BE3 FW.
3495          * Also, this is done by FW in Lancer chip.
3496          */
3497         if (be_max_vfs(adapter) && !pci_num_vf(adapter->pdev)) {
3498                 status = be_cmd_set_sriov_config(adapter,
3499                                                  adapter->pool_res,
3500                                                  adapter->num_vfs);
3501                 if (status)
3502                         dev_err(dev, "Failed to optimize SR-IOV resources\n");
3503         }
3504 }
3505
3506 static int be_get_config(struct be_adapter *adapter)
3507 {
3508         u16 profile_id;
3509         int status;
3510
3511         status = be_cmd_query_fw_cfg(adapter);
3512         if (status)
3513                 return status;
3514
3515          if (be_physfn(adapter)) {
3516                 status = be_cmd_get_active_profile(adapter, &profile_id);
3517                 if (!status)
3518                         dev_info(&adapter->pdev->dev,
3519                                  "Using profile 0x%x\n", profile_id);
3520         }
3521
3522         if (!BE2_chip(adapter) && be_physfn(adapter))
3523                 be_sriov_config(adapter);
3524
3525         status = be_get_resources(adapter);
3526         if (status)
3527                 return status;
3528
3529         adapter->pmac_id = kcalloc(be_max_uc(adapter),
3530                                    sizeof(*adapter->pmac_id), GFP_KERNEL);
3531         if (!adapter->pmac_id)
3532                 return -ENOMEM;
3533
3534         /* Sanitize cfg_num_qs based on HW and platform limits */
3535         adapter->cfg_num_qs = min(adapter->cfg_num_qs, be_max_qs(adapter));
3536
3537         return 0;
3538 }
3539
3540 static int be_mac_setup(struct be_adapter *adapter)
3541 {
3542         u8 mac[ETH_ALEN];
3543         int status;
3544
3545         if (is_zero_ether_addr(adapter->netdev->dev_addr)) {
3546                 status = be_cmd_get_perm_mac(adapter, mac);
3547                 if (status)
3548                         return status;
3549
3550                 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN);
3551                 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN);
3552         } else {
3553                 /* Maybe the HW was reset; dev_addr must be re-programmed */
3554                 memcpy(mac, adapter->netdev->dev_addr, ETH_ALEN);
3555         }
3556
3557         /* For BE3-R VFs, the PF programs the initial MAC address */
3558         if (!(BEx_chip(adapter) && be_virtfn(adapter)))
3559                 be_cmd_pmac_add(adapter, mac, adapter->if_handle,
3560                                 &adapter->pmac_id[0], 0);
3561         return 0;
3562 }
3563
3564 static void be_schedule_worker(struct be_adapter *adapter)
3565 {
3566         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
3567         adapter->flags |= BE_FLAGS_WORKER_SCHEDULED;
3568 }
3569
3570 static int be_setup_queues(struct be_adapter *adapter)
3571 {
3572         struct net_device *netdev = adapter->netdev;
3573         int status;
3574
3575         status = be_evt_queues_create(adapter);
3576         if (status)
3577                 goto err;
3578
3579         status = be_tx_qs_create(adapter);
3580         if (status)
3581                 goto err;
3582
3583         status = be_rx_cqs_create(adapter);
3584         if (status)
3585                 goto err;
3586
3587         status = be_mcc_queues_create(adapter);
3588         if (status)
3589                 goto err;
3590
3591         status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs);
3592         if (status)
3593                 goto err;
3594
3595         status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs);
3596         if (status)
3597                 goto err;
3598
3599         return 0;
3600 err:
3601         dev_err(&adapter->pdev->dev, "queue_setup failed\n");
3602         return status;
3603 }
3604
3605 int be_update_queues(struct be_adapter *adapter)
3606 {
3607         struct net_device *netdev = adapter->netdev;
3608         int status;
3609
3610         if (netif_running(netdev))
3611                 be_close(netdev);
3612
3613         be_cancel_worker(adapter);
3614
3615         /* If any vectors have been shared with RoCE we cannot re-program
3616          * the MSIx table.
3617          */
3618         if (!adapter->num_msix_roce_vec)
3619                 be_msix_disable(adapter);
3620
3621         be_clear_queues(adapter);
3622
3623         if (!msix_enabled(adapter)) {
3624                 status = be_msix_enable(adapter);
3625                 if (status)
3626                         return status;
3627         }
3628
3629         status = be_setup_queues(adapter);
3630         if (status)
3631                 return status;
3632
3633         be_schedule_worker(adapter);
3634
3635         if (netif_running(netdev))
3636                 status = be_open(netdev);
3637
3638         return status;
3639 }
3640
3641 static int be_setup(struct be_adapter *adapter)
3642 {
3643         struct device *dev = &adapter->pdev->dev;
3644         u32 tx_fc, rx_fc, en_flags;
3645         int status;
3646
3647         be_setup_init(adapter);
3648
3649         if (!lancer_chip(adapter))
3650                 be_cmd_req_native_mode(adapter);
3651
3652         status = be_get_config(adapter);
3653         if (status)
3654                 goto err;
3655
3656         status = be_msix_enable(adapter);
3657         if (status)
3658                 goto err;
3659
3660         en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST |
3661                    BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS;
3662         if (adapter->function_caps & BE_FUNCTION_CAPS_RSS)
3663                 en_flags |= BE_IF_FLAGS_RSS;
3664         en_flags = en_flags & be_if_cap_flags(adapter);
3665         status = be_cmd_if_create(adapter, be_if_cap_flags(adapter), en_flags,
3666                                   &adapter->if_handle, 0);
3667         if (status)
3668                 goto err;
3669
3670         /* Updating real_num_tx/rx_queues() requires rtnl_lock() */
3671         rtnl_lock();
3672         status = be_setup_queues(adapter);
3673         rtnl_unlock();
3674         if (status)
3675                 goto err;
3676
3677         be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0);
3678
3679         status = be_mac_setup(adapter);
3680         if (status)
3681                 goto err;
3682
3683         be_cmd_get_fw_ver(adapter);
3684         dev_info(dev, "FW version is %s\n", adapter->fw_ver);
3685
3686         if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) {
3687                 dev_err(dev, "Firmware on card is old(%s), IRQs may not work",
3688                         adapter->fw_ver);
3689                 dev_err(dev, "Please upgrade firmware to version >= 4.0\n");
3690         }
3691
3692         if (adapter->vlans_added)
3693                 be_vid_config(adapter);
3694
3695         be_set_rx_mode(adapter->netdev);
3696
3697         be_cmd_get_acpi_wol_cap(adapter);
3698
3699         be_cmd_get_flow_control(adapter, &tx_fc, &rx_fc);
3700
3701         if (rx_fc != adapter->rx_fc || tx_fc != adapter->tx_fc)
3702                 be_cmd_set_flow_control(adapter, adapter->tx_fc,
3703                                         adapter->rx_fc);
3704
3705         if (be_physfn(adapter))
3706                 be_cmd_set_logical_link_config(adapter,
3707                                                IFLA_VF_LINK_STATE_AUTO, 0);
3708
3709         if (adapter->num_vfs)
3710                 be_vf_setup(adapter);
3711
3712         status = be_cmd_get_phy_info(adapter);
3713         if (!status && be_pause_supported(adapter))
3714                 adapter->phy.fc_autoneg = 1;
3715
3716         be_schedule_worker(adapter);
3717         adapter->flags |= BE_FLAGS_SETUP_DONE;
3718         return 0;
3719 err:
3720         be_clear(adapter);
3721         return status;
3722 }
3723
3724 #ifdef CONFIG_NET_POLL_CONTROLLER
3725 static void be_netpoll(struct net_device *netdev)
3726 {
3727         struct be_adapter *adapter = netdev_priv(netdev);
3728         struct be_eq_obj *eqo;
3729         int i;
3730
3731         for_all_evt_queues(adapter, eqo, i) {
3732                 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0);
3733                 napi_schedule(&eqo->napi);
3734         }
3735 }
3736 #endif
3737
3738 static char flash_cookie[2][16] = {"*** SE FLAS", "H DIRECTORY *** "};
3739
3740 static bool phy_flashing_required(struct be_adapter *adapter)
3741 {
3742         return (adapter->phy.phy_type == TN_8022 &&
3743                 adapter->phy.interface_type == PHY_TYPE_BASET_10GB);
3744 }
3745
3746 static bool is_comp_in_ufi(struct be_adapter *adapter,
3747                            struct flash_section_info *fsec, int type)
3748 {
3749         int i = 0, img_type = 0;
3750         struct flash_section_info_g2 *fsec_g2 = NULL;
3751
3752         if (BE2_chip(adapter))
3753                 fsec_g2 = (struct flash_section_info_g2 *)fsec;
3754
3755         for (i = 0; i < MAX_FLASH_COMP; i++) {
3756                 if (fsec_g2)
3757                         img_type = le32_to_cpu(fsec_g2->fsec_entry[i].type);
3758                 else
3759                         img_type = le32_to_cpu(fsec->fsec_entry[i].type);
3760
3761                 if (img_type == type)
3762                         return true;
3763         }
3764         return false;
3765
3766 }
3767
3768 static struct flash_section_info *get_fsec_info(struct be_adapter *adapter,
3769                                                 int header_size,
3770                                                 const struct firmware *fw)
3771 {
3772         struct flash_section_info *fsec = NULL;
3773         const u8 *p = fw->data;
3774
3775         p += header_size;
3776         while (p < (fw->data + fw->size)) {
3777                 fsec = (struct flash_section_info *)p;
3778                 if (!memcmp(flash_cookie, fsec->cookie, sizeof(flash_cookie)))
3779                         return fsec;
3780                 p += 32;
3781         }
3782         return NULL;
3783 }
3784
3785 static int be_check_flash_crc(struct be_adapter *adapter, const u8 *p,
3786                               u32 img_offset, u32 img_size, int hdr_size,
3787                               u16 img_optype, bool *crc_match)
3788 {
3789         u32 crc_offset;
3790         int status;
3791         u8 crc[4];
3792
3793         status = be_cmd_get_flash_crc(adapter, crc, img_optype, img_size - 4);
3794         if (status)
3795                 return status;
3796
3797         crc_offset = hdr_size + img_offset + img_size - 4;
3798
3799         /* Skip flashing, if crc of flashed region matches */
3800         if (!memcmp(crc, p + crc_offset, 4))
3801                 *crc_match = true;
3802         else
3803                 *crc_match = false;
3804
3805         return status;
3806 }
3807
3808 static int be_flash(struct be_adapter *adapter, const u8 *img,
3809                     struct be_dma_mem *flash_cmd, int optype, int img_size)
3810 {
3811         struct be_cmd_write_flashrom *req = flash_cmd->va;
3812         u32 total_bytes, flash_op, num_bytes;
3813         int status;
3814
3815         total_bytes = img_size;
3816         while (total_bytes) {
3817                 num_bytes = min_t(u32, 32*1024, total_bytes);
3818
3819                 total_bytes -= num_bytes;
3820
3821                 if (!total_bytes) {
3822                         if (optype == OPTYPE_PHY_FW)
3823                                 flash_op = FLASHROM_OPER_PHY_FLASH;
3824                         else
3825                                 flash_op = FLASHROM_OPER_FLASH;
3826                 } else {
3827                         if (optype == OPTYPE_PHY_FW)
3828                                 flash_op = FLASHROM_OPER_PHY_SAVE;
3829                         else
3830                                 flash_op = FLASHROM_OPER_SAVE;
3831                 }
3832
3833                 memcpy(req->data_buf, img, num_bytes);
3834                 img += num_bytes;
3835                 status = be_cmd_write_flashrom(adapter, flash_cmd, optype,
3836                                                flash_op, num_bytes);
3837                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST &&
3838                     optype == OPTYPE_PHY_FW)
3839                         break;
3840                 else if (status)
3841                         return status;
3842         }
3843         return 0;
3844 }
3845
3846 /* For BE2, BE3 and BE3-R */
3847 static int be_flash_BEx(struct be_adapter *adapter,
3848                         const struct firmware *fw,
3849                         struct be_dma_mem *flash_cmd, int num_of_images)
3850 {
3851         int img_hdrs_size = (num_of_images * sizeof(struct image_hdr));
3852         struct device *dev = &adapter->pdev->dev;
3853         struct flash_section_info *fsec = NULL;
3854         int status, i, filehdr_size, num_comp;
3855         const struct flash_comp *pflashcomp;
3856         bool crc_match;
3857         const u8 *p;
3858
3859         struct flash_comp gen3_flash_types[] = {
3860                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, OPTYPE_ISCSI_ACTIVE,
3861                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_iSCSI},
3862                 { FLASH_REDBOOT_START_g3, OPTYPE_REDBOOT,
3863                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g3, IMAGE_BOOT_CODE},
3864                 { FLASH_iSCSI_BIOS_START_g3, OPTYPE_BIOS,
3865                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_ISCSI},
3866                 { FLASH_PXE_BIOS_START_g3, OPTYPE_PXE_BIOS,
3867                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_PXE},
3868                 { FLASH_FCoE_BIOS_START_g3, OPTYPE_FCOE_BIOS,
3869                         FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_FCoE},
3870                 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, OPTYPE_ISCSI_BACKUP,
3871                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_iSCSI},
3872                 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, OPTYPE_FCOE_FW_ACTIVE,
3873                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_FCoE},
3874                 { FLASH_FCoE_BACKUP_IMAGE_START_g3, OPTYPE_FCOE_FW_BACKUP,
3875                         FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_FCoE},
3876                 { FLASH_NCSI_START_g3, OPTYPE_NCSI_FW,
3877                         FLASH_NCSI_IMAGE_MAX_SIZE_g3, IMAGE_NCSI},
3878                 { FLASH_PHY_FW_START_g3, OPTYPE_PHY_FW,
3879                         FLASH_PHY_FW_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_PHY}
3880         };
3881
3882         struct flash_comp gen2_flash_types[] = {
3883                 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, OPTYPE_ISCSI_ACTIVE,
3884                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_iSCSI},
3885                 { FLASH_REDBOOT_START_g2, OPTYPE_REDBOOT,
3886                         FLASH_REDBOOT_IMAGE_MAX_SIZE_g2, IMAGE_BOOT_CODE},
3887                 { FLASH_iSCSI_BIOS_START_g2, OPTYPE_BIOS,
3888                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_ISCSI},
3889                 { FLASH_PXE_BIOS_START_g2, OPTYPE_PXE_BIOS,
3890                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_PXE},
3891                 { FLASH_FCoE_BIOS_START_g2, OPTYPE_FCOE_BIOS,
3892                         FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_FCoE},
3893                 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, OPTYPE_ISCSI_BACKUP,
3894                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_iSCSI},
3895                 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, OPTYPE_FCOE_FW_ACTIVE,
3896                         FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_FCoE},
3897                 { FLASH_FCoE_BACKUP_IMAGE_START_g2, OPTYPE_FCOE_FW_BACKUP,
3898                          FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_FCoE}
3899         };
3900
3901         if (BE3_chip(adapter)) {
3902                 pflashcomp = gen3_flash_types;
3903                 filehdr_size = sizeof(struct flash_file_hdr_g3);
3904                 num_comp = ARRAY_SIZE(gen3_flash_types);
3905         } else {
3906                 pflashcomp = gen2_flash_types;
3907                 filehdr_size = sizeof(struct flash_file_hdr_g2);
3908                 num_comp = ARRAY_SIZE(gen2_flash_types);
3909         }
3910
3911         /* Get flash section info*/
3912         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
3913         if (!fsec) {
3914                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
3915                 return -1;
3916         }
3917         for (i = 0; i < num_comp; i++) {
3918                 if (!is_comp_in_ufi(adapter, fsec, pflashcomp[i].img_type))
3919                         continue;
3920
3921                 if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) &&
3922                     memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0)
3923                         continue;
3924
3925                 if (pflashcomp[i].optype == OPTYPE_PHY_FW  &&
3926                     !phy_flashing_required(adapter))
3927                                 continue;
3928
3929                 if (pflashcomp[i].optype == OPTYPE_REDBOOT) {
3930                         status = be_check_flash_crc(adapter, fw->data,
3931                                                     pflashcomp[i].offset,
3932                                                     pflashcomp[i].size,
3933                                                     filehdr_size +
3934                                                     img_hdrs_size,
3935                                                     OPTYPE_REDBOOT, &crc_match);
3936                         if (status) {
3937                                 dev_err(dev,
3938                                         "Could not get CRC for 0x%x region\n",
3939                                         pflashcomp[i].optype);
3940                                 continue;
3941                         }
3942
3943                         if (crc_match)
3944                                 continue;
3945                 }
3946
3947                 p = fw->data + filehdr_size + pflashcomp[i].offset +
3948                         img_hdrs_size;
3949                 if (p + pflashcomp[i].size > fw->data + fw->size)
3950                         return -1;
3951
3952                 status = be_flash(adapter, p, flash_cmd, pflashcomp[i].optype,
3953                                   pflashcomp[i].size);
3954                 if (status) {
3955                         dev_err(dev, "Flashing section type 0x%x failed\n",
3956                                 pflashcomp[i].img_type);
3957                         return status;
3958                 }
3959         }
3960         return 0;
3961 }
3962
3963 static u16 be_get_img_optype(struct flash_section_entry fsec_entry)
3964 {
3965         u32 img_type = le32_to_cpu(fsec_entry.type);
3966         u16 img_optype = le16_to_cpu(fsec_entry.optype);
3967
3968         if (img_optype != 0xFFFF)
3969                 return img_optype;
3970
3971         switch (img_type) {
3972         case IMAGE_FIRMWARE_iSCSI:
3973                 img_optype = OPTYPE_ISCSI_ACTIVE;
3974                 break;
3975         case IMAGE_BOOT_CODE:
3976                 img_optype = OPTYPE_REDBOOT;
3977                 break;
3978         case IMAGE_OPTION_ROM_ISCSI:
3979                 img_optype = OPTYPE_BIOS;
3980                 break;
3981         case IMAGE_OPTION_ROM_PXE:
3982                 img_optype = OPTYPE_PXE_BIOS;
3983                 break;
3984         case IMAGE_OPTION_ROM_FCoE:
3985                 img_optype = OPTYPE_FCOE_BIOS;
3986                 break;
3987         case IMAGE_FIRMWARE_BACKUP_iSCSI:
3988                 img_optype = OPTYPE_ISCSI_BACKUP;
3989                 break;
3990         case IMAGE_NCSI:
3991                 img_optype = OPTYPE_NCSI_FW;
3992                 break;
3993         case IMAGE_FLASHISM_JUMPVECTOR:
3994                 img_optype = OPTYPE_FLASHISM_JUMPVECTOR;
3995                 break;
3996         case IMAGE_FIRMWARE_PHY:
3997                 img_optype = OPTYPE_SH_PHY_FW;
3998                 break;
3999         case IMAGE_REDBOOT_DIR:
4000                 img_optype = OPTYPE_REDBOOT_DIR;
4001                 break;
4002         case IMAGE_REDBOOT_CONFIG:
4003                 img_optype = OPTYPE_REDBOOT_CONFIG;
4004                 break;
4005         case IMAGE_UFI_DIR:
4006                 img_optype = OPTYPE_UFI_DIR;
4007                 break;
4008         default:
4009                 break;
4010         }
4011
4012         return img_optype;
4013 }
4014
4015 static int be_flash_skyhawk(struct be_adapter *adapter,
4016                             const struct firmware *fw,
4017                             struct be_dma_mem *flash_cmd, int num_of_images)
4018 {
4019         int img_hdrs_size = num_of_images * sizeof(struct image_hdr);
4020         struct device *dev = &adapter->pdev->dev;
4021         struct flash_section_info *fsec = NULL;
4022         u32 img_offset, img_size, img_type;
4023         int status, i, filehdr_size;
4024         bool crc_match, old_fw_img;
4025         u16 img_optype;
4026         const u8 *p;
4027
4028         filehdr_size = sizeof(struct flash_file_hdr_g3);
4029         fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw);
4030         if (!fsec) {
4031                 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n");
4032                 return -EINVAL;
4033         }
4034
4035         for (i = 0; i < le32_to_cpu(fsec->fsec_hdr.num_images); i++) {
4036                 img_offset = le32_to_cpu(fsec->fsec_entry[i].offset);
4037                 img_size   = le32_to_cpu(fsec->fsec_entry[i].pad_size);
4038                 img_type   = le32_to_cpu(fsec->fsec_entry[i].type);
4039                 img_optype = be_get_img_optype(fsec->fsec_entry[i]);
4040                 old_fw_img = fsec->fsec_entry[i].optype == 0xFFFF;
4041
4042                 if (img_optype == 0xFFFF)
4043                         continue;
4044                 /* Don't bother verifying CRC if an old FW image is being
4045                  * flashed
4046                  */
4047                 if (old_fw_img)
4048                         goto flash;
4049
4050                 status = be_check_flash_crc(adapter, fw->data, img_offset,
4051                                             img_size, filehdr_size +
4052                                             img_hdrs_size, img_optype,
4053                                             &crc_match);
4054                 /* The current FW image on the card does not recognize the new
4055                  * FLASH op_type. The FW download is partially complete.
4056                  * Reboot the server now to enable FW image to recognize the
4057                  * new FLASH op_type. To complete the remaining process,
4058                  * download the same FW again after the reboot.
4059                  */
4060                 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST ||
4061                     base_status(status) == MCC_STATUS_ILLEGAL_FIELD) {
4062                         dev_err(dev, "Flash incomplete. Reset the server\n");
4063                         dev_err(dev, "Download FW image again after reset\n");
4064                         return -EAGAIN;
4065                 } else if (status) {
4066                         dev_err(dev, "Could not get CRC for 0x%x region\n",
4067                                 img_optype);
4068                         return -EFAULT;
4069                 }
4070
4071                 if (crc_match)
4072                         continue;
4073
4074 flash:
4075                 p = fw->data + filehdr_size + img_offset + img_hdrs_size;
4076                 if (p + img_size > fw->data + fw->size)
4077                         return -1;
4078
4079                 status = be_flash(adapter, p, flash_cmd, img_optype, img_size);
4080                 /* For old FW images ignore ILLEGAL_FIELD error or errors on
4081                  * UFI_DIR region
4082                  */
4083                 if (old_fw_img &&
4084                     (base_status(status) == MCC_STATUS_ILLEGAL_FIELD ||
4085                      (img_optype == OPTYPE_UFI_DIR &&
4086                       base_status(status) == MCC_STATUS_FAILED))) {
4087                         continue;
4088                 } else if (status) {
4089                         dev_err(dev, "Flashing section type 0x%x failed\n",
4090                                 img_type);
4091                         return -EFAULT;
4092                 }
4093         }
4094         return 0;
4095 }
4096
4097 static int lancer_fw_download(struct be_adapter *adapter,
4098                               const struct firmware *fw)
4099 {
4100 #define LANCER_FW_DOWNLOAD_CHUNK      (32 * 1024)
4101 #define LANCER_FW_DOWNLOAD_LOCATION   "/prg"
4102         struct device *dev = &adapter->pdev->dev;
4103         struct be_dma_mem flash_cmd;
4104         const u8 *data_ptr = NULL;
4105         u8 *dest_image_ptr = NULL;
4106         size_t image_size = 0;
4107         u32 chunk_size = 0;
4108         u32 data_written = 0;
4109         u32 offset = 0;
4110         int status = 0;
4111         u8 add_status = 0;
4112         u8 change_status;
4113
4114         if (!IS_ALIGNED(fw->size, sizeof(u32))) {
4115                 dev_err(dev, "FW image size should be multiple of 4\n");
4116                 return -EINVAL;
4117         }
4118
4119         flash_cmd.size = sizeof(struct lancer_cmd_req_write_object)
4120                                 + LANCER_FW_DOWNLOAD_CHUNK;
4121         flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size,
4122                                           &flash_cmd.dma, GFP_KERNEL);
4123         if (!flash_cmd.va)
4124                 return -ENOMEM;
4125
4126         dest_image_ptr = flash_cmd.va +
4127                                 sizeof(struct lancer_cmd_req_write_object);
4128         image_size = fw->size;
4129         data_ptr = fw->data;
4130
4131         while (image_size) {
4132                 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK);
4133
4134                 /* Copy the image chunk content. */
4135                 memcpy(dest_image_ptr, data_ptr, chunk_size);
4136
4137                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4138                                                  chunk_size, offset,
4139                                                  LANCER_FW_DOWNLOAD_LOCATION,
4140                                                  &data_written, &change_status,
4141                                                  &add_status);
4142                 if (status)
4143                         break;
4144
4145                 offset += data_written;
4146                 data_ptr += data_written;
4147                 image_size -= data_written;
4148         }
4149
4150         if (!status) {
4151                 /* Commit the FW written */
4152                 status = lancer_cmd_write_object(adapter, &flash_cmd,
4153                                                  0, offset,
4154                                                  LANCER_FW_DOWNLOAD_LOCATION,
4155                                                  &data_written, &change_status,
4156                                                  &add_status);
4157         }
4158
4159         dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma);
4160         if (status) {
4161                 dev_err(dev, "Firmware load error\n");
4162                 return be_cmd_status(status);
4163         }
4164
4165         dev_info(dev, "Firmware flashed successfully\n");
4166
4167         if (change_status == LANCER_FW_RESET_NEEDED) {
4168                 dev_info(dev, "Resetting adapter to activate new FW\n");
4169                 status = lancer_physdev_ctrl(adapter,
4170                                              PHYSDEV_CONTROL_FW_RESET_MASK);
4171                 if (status) {
4172                         dev_err(dev, "Adapter busy, could not reset FW\n");
4173                         dev_err(dev, "Reboot server to activate new FW\n");
4174                 }
4175         } else if (change_status != LANCER_NO_RESET_NEEDED) {
4176                 dev_info(dev, "Reboot server to activate new FW\n");
4177         }
4178
4179         return 0;
4180 }
4181
4182 #define UFI_TYPE2               2
4183 #define UFI_TYPE3               3
4184 #define UFI_TYPE3R              10
4185 #define UFI_TYPE4               4
4186 static int be_get_ufi_type(struct be_adapter *adapter,
4187                            struct flash_file_hdr_g3 *fhdr)
4188 {
4189         if (!fhdr)
4190                 goto be_get_ufi_exit;
4191
4192         if (skyhawk_chip(adapter) && fhdr->build[0] == '4')
4193                 return UFI_TYPE4;
4194         else if (BE3_chip(adapter) && fhdr->build[0] == '3') {
4195                 if (fhdr->asic_type_rev == 0x10)
4196                         return UFI_TYPE3R;
4197                 else
4198                         return UFI_TYPE3;
4199         } else if (BE2_chip(adapter) && fhdr->build[0] == '2')
4200                 return UFI_TYPE2;
4201
4202 be_get_ufi_exit:
4203         dev_err(&adapter->pdev->dev,
4204                 "UFI and Interface are not compatible for flashing\n");
4205         return -1;
4206 }
4207
4208 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw)
4209 {
4210         struct flash_file_hdr_g3 *fhdr3;
4211         struct image_hdr *img_hdr_ptr = NULL;
4212         struct be_dma_mem flash_cmd;
4213         const u8 *p;
4214         int status = 0, i = 0, num_imgs = 0, ufi_type = 0;
4215
4216         flash_cmd.size = sizeof(struct be_cmd_write_flashrom);
4217         flash_cmd.va = dma_alloc_coherent(&adapter->pdev->dev, flash_cmd.size,
4218                                           &flash_cmd.dma, GFP_KERNEL);
4219         if (!flash_cmd.va) {
4220                 status = -ENOMEM;
4221                 goto be_fw_exit;
4222         }
4223
4224         p = fw->data;
4225         fhdr3 = (struct flash_file_hdr_g3 *)p;
4226
4227         ufi_type = be_get_ufi_type(adapter, fhdr3);
4228
4229         num_imgs = le32_to_cpu(fhdr3->num_imgs);
4230         for (i = 0; i < num_imgs; i++) {
4231                 img_hdr_ptr = (struct image_hdr *)(fw->data +
4232                                 (sizeof(struct flash_file_hdr_g3) +
4233                                  i * sizeof(struct image_hdr)));
4234                 if (le32_to_cpu(img_hdr_ptr->imageid) == 1) {
4235                         switch (ufi_type) {
4236                         case UFI_TYPE4:
4237                                 status = be_flash_skyhawk(adapter, fw,
4238                                                           &flash_cmd, num_imgs);
4239                                 break;
4240                         case UFI_TYPE3R:
4241                                 status = be_flash_BEx(adapter, fw, &flash_cmd,
4242                                                       num_imgs);
4243                                 break;
4244                         case UFI_TYPE3:
4245                                 /* Do not flash this ufi on BE3-R cards */
4246                                 if (adapter->asic_rev < 0x10)
4247                                         status = be_flash_BEx(adapter, fw,
4248                                                               &flash_cmd,
4249                                                               num_imgs);
4250                                 else {
4251                                         status = -EINVAL;
4252                                         dev_err(&adapter->pdev->dev,
4253                                                 "Can't load BE3 UFI on BE3R\n");
4254                                 }
4255                         }
4256                 }
4257         }
4258
4259         if (ufi_type == UFI_TYPE2)
4260                 status = be_flash_BEx(adapter, fw, &flash_cmd, 0);
4261         else if (ufi_type == -1)
4262                 status = -EINVAL;
4263
4264         dma_free_coherent(&adapter->pdev->dev, flash_cmd.size, flash_cmd.va,
4265                           flash_cmd.dma);
4266         if (status) {
4267                 dev_err(&adapter->pdev->dev, "Firmware load error\n");
4268                 goto be_fw_exit;
4269         }
4270
4271         dev_info(&adapter->pdev->dev, "Firmware flashed successfully\n");
4272
4273 be_fw_exit:
4274         return status;
4275 }
4276
4277 int be_load_fw(struct be_adapter *adapter, u8 *fw_file)
4278 {
4279         const struct firmware *fw;
4280         int status;
4281
4282         if (!netif_running(adapter->netdev)) {
4283                 dev_err(&adapter->pdev->dev,
4284                         "Firmware load not allowed (interface is down)\n");
4285                 return -ENETDOWN;
4286         }
4287
4288         status = request_firmware(&fw, fw_file, &adapter->pdev->dev);
4289         if (status)
4290                 goto fw_exit;
4291
4292         dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file);
4293
4294         if (lancer_chip(adapter))
4295                 status = lancer_fw_download(adapter, fw);
4296         else
4297                 status = be_fw_download(adapter, fw);
4298
4299         if (!status)
4300                 be_cmd_get_fw_ver(adapter);
4301
4302 fw_exit:
4303         release_firmware(fw);
4304         return status;
4305 }
4306
4307 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh)
4308 {
4309         struct be_adapter *adapter = netdev_priv(dev);
4310         struct nlattr *attr, *br_spec;
4311         int rem;
4312         int status = 0;
4313         u16 mode = 0;
4314
4315         if (!sriov_enabled(adapter))
4316                 return -EOPNOTSUPP;
4317
4318         br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
4319         if (!br_spec)
4320                 return -EINVAL;
4321
4322         nla_for_each_nested(attr, br_spec, rem) {
4323                 if (nla_type(attr) != IFLA_BRIDGE_MODE)
4324                         continue;
4325
4326                 if (nla_len(attr) < sizeof(mode))
4327                         return -EINVAL;
4328
4329                 mode = nla_get_u16(attr);
4330                 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB)
4331                         return -EINVAL;
4332
4333                 status = be_cmd_set_hsw_config(adapter, 0, 0,
4334                                                adapter->if_handle,
4335                                                mode == BRIDGE_MODE_VEPA ?
4336                                                PORT_FWD_TYPE_VEPA :
4337                                                PORT_FWD_TYPE_VEB);
4338                 if (status)
4339                         goto err;
4340
4341                 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n",
4342                          mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4343
4344                 return status;
4345         }
4346 err:
4347         dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n",
4348                 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB");
4349
4350         return status;
4351 }
4352
4353 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
4354                                  struct net_device *dev, u32 filter_mask)
4355 {
4356         struct be_adapter *adapter = netdev_priv(dev);
4357         int status = 0;
4358         u8 hsw_mode;
4359
4360         if (!sriov_enabled(adapter))
4361                 return 0;
4362
4363         /* BE and Lancer chips support VEB mode only */
4364         if (BEx_chip(adapter) || lancer_chip(adapter)) {
4365                 hsw_mode = PORT_FWD_TYPE_VEB;
4366         } else {
4367                 status = be_cmd_get_hsw_config(adapter, NULL, 0,
4368                                                adapter->if_handle, &hsw_mode);
4369                 if (status)
4370                         return 0;
4371         }
4372
4373         return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
4374                                        hsw_mode == PORT_FWD_TYPE_VEPA ?
4375                                        BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB,
4376                                        0, 0);
4377 }
4378
4379 #ifdef CONFIG_BE2NET_VXLAN
4380 /* VxLAN offload Notes:
4381  *
4382  * The stack defines tunnel offload flags (hw_enc_features) for IP and doesn't
4383  * distinguish various types of transports (VxLAN, GRE, NVGRE ..). So, offload
4384  * is expected to work across all types of IP tunnels once exported. Skyhawk
4385  * supports offloads for either VxLAN or NVGRE, exclusively. So we export VxLAN
4386  * offloads in hw_enc_features only when a VxLAN port is added. If other (non
4387  * VxLAN) tunnels are configured while VxLAN offloads are enabled, offloads for
4388  * those other tunnels are unexported on the fly through ndo_features_check().
4389  *
4390  * Skyhawk supports VxLAN offloads only for one UDP dport. So, if the stack
4391  * adds more than one port, disable offloads and don't re-enable them again
4392  * until after all the tunnels are removed.
4393  */
4394 static void be_add_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4395                               __be16 port)
4396 {
4397         struct be_adapter *adapter = netdev_priv(netdev);
4398         struct device *dev = &adapter->pdev->dev;
4399         int status;
4400
4401         if (lancer_chip(adapter) || BEx_chip(adapter))
4402                 return;
4403
4404         if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) {
4405                 dev_info(dev,
4406                          "Only one UDP port supported for VxLAN offloads\n");
4407                 dev_info(dev, "Disabling VxLAN offloads\n");
4408                 adapter->vxlan_port_count++;
4409                 goto err;
4410         }
4411
4412         if (adapter->vxlan_port_count++ >= 1)
4413                 return;
4414
4415         status = be_cmd_manage_iface(adapter, adapter->if_handle,
4416                                      OP_CONVERT_NORMAL_TO_TUNNEL);
4417         if (status) {
4418                 dev_warn(dev, "Failed to convert normal interface to tunnel\n");
4419                 goto err;
4420         }
4421
4422         status = be_cmd_set_vxlan_port(adapter, port);
4423         if (status) {
4424                 dev_warn(dev, "Failed to add VxLAN port\n");
4425                 goto err;
4426         }
4427         adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS;
4428         adapter->vxlan_port = port;
4429
4430         netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4431                                    NETIF_F_TSO | NETIF_F_TSO6 |
4432                                    NETIF_F_GSO_UDP_TUNNEL;
4433         netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL;
4434         netdev->features |= NETIF_F_GSO_UDP_TUNNEL;
4435
4436         dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n",
4437                  be16_to_cpu(port));
4438         return;
4439 err:
4440         be_disable_vxlan_offloads(adapter);
4441 }
4442
4443 static void be_del_vxlan_port(struct net_device *netdev, sa_family_t sa_family,
4444                               __be16 port)
4445 {
4446         struct be_adapter *adapter = netdev_priv(netdev);
4447
4448         if (lancer_chip(adapter) || BEx_chip(adapter))
4449                 return;
4450
4451         if (adapter->vxlan_port != port)
4452                 goto done;
4453
4454         be_disable_vxlan_offloads(adapter);
4455
4456         dev_info(&adapter->pdev->dev,
4457                  "Disabled VxLAN offloads for UDP port %d\n",
4458                  be16_to_cpu(port));
4459 done:
4460         adapter->vxlan_port_count--;
4461 }
4462
4463 static netdev_features_t be_features_check(struct sk_buff *skb,
4464                                            struct net_device *dev,
4465                                            netdev_features_t features)
4466 {
4467         struct be_adapter *adapter = netdev_priv(dev);
4468         u8 l4_hdr = 0;
4469
4470         /* The code below restricts offload features for some tunneled packets.
4471          * Offload features for normal (non tunnel) packets are unchanged.
4472          */
4473         if (!skb->encapsulation ||
4474             !(adapter->flags & BE_FLAGS_VXLAN_OFFLOADS))
4475                 return features;
4476
4477         /* It's an encapsulated packet and VxLAN offloads are enabled. We
4478          * should disable tunnel offload features if it's not a VxLAN packet,
4479          * as tunnel offloads have been enabled only for VxLAN. This is done to
4480          * allow other tunneled traffic like GRE work fine while VxLAN
4481          * offloads are configured in Skyhawk-R.
4482          */
4483         switch (vlan_get_protocol(skb)) {
4484         case htons(ETH_P_IP):
4485                 l4_hdr = ip_hdr(skb)->protocol;
4486                 break;
4487         case htons(ETH_P_IPV6):
4488                 l4_hdr = ipv6_hdr(skb)->nexthdr;
4489                 break;
4490         default:
4491                 return features;
4492         }
4493
4494         if (l4_hdr != IPPROTO_UDP ||
4495             skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
4496             skb->inner_protocol != htons(ETH_P_TEB) ||
4497             skb_inner_mac_header(skb) - skb_transport_header(skb) !=
4498             sizeof(struct udphdr) + sizeof(struct vxlanhdr))
4499                 return features & ~(NETIF_F_ALL_CSUM | NETIF_F_GSO_MASK);
4500
4501         return features;
4502 }
4503 #endif
4504
4505 static const struct net_device_ops be_netdev_ops = {
4506         .ndo_open               = be_open,
4507         .ndo_stop               = be_close,
4508         .ndo_start_xmit         = be_xmit,
4509         .ndo_set_rx_mode        = be_set_rx_mode,
4510         .ndo_set_mac_address    = be_mac_addr_set,
4511         .ndo_change_mtu         = be_change_mtu,
4512         .ndo_get_stats64        = be_get_stats64,
4513         .ndo_validate_addr      = eth_validate_addr,
4514         .ndo_vlan_rx_add_vid    = be_vlan_add_vid,
4515         .ndo_vlan_rx_kill_vid   = be_vlan_rem_vid,
4516         .ndo_set_vf_mac         = be_set_vf_mac,
4517         .ndo_set_vf_vlan        = be_set_vf_vlan,
4518         .ndo_set_vf_rate        = be_set_vf_tx_rate,
4519         .ndo_get_vf_config      = be_get_vf_config,
4520         .ndo_set_vf_link_state  = be_set_vf_link_state,
4521 #ifdef CONFIG_NET_POLL_CONTROLLER
4522         .ndo_poll_controller    = be_netpoll,
4523 #endif
4524         .ndo_bridge_setlink     = be_ndo_bridge_setlink,
4525         .ndo_bridge_getlink     = be_ndo_bridge_getlink,
4526 #ifdef CONFIG_NET_RX_BUSY_POLL
4527         .ndo_busy_poll          = be_busy_poll,
4528 #endif
4529 #ifdef CONFIG_BE2NET_VXLAN
4530         .ndo_add_vxlan_port     = be_add_vxlan_port,
4531         .ndo_del_vxlan_port     = be_del_vxlan_port,
4532         .ndo_features_check     = be_features_check,
4533 #endif
4534 };
4535
4536 static void be_netdev_init(struct net_device *netdev)
4537 {
4538         struct be_adapter *adapter = netdev_priv(netdev);
4539
4540         netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4541                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
4542                 NETIF_F_HW_VLAN_CTAG_TX;
4543         if (be_multi_rxq(adapter))
4544                 netdev->hw_features |= NETIF_F_RXHASH;
4545
4546         netdev->features |= netdev->hw_features |
4547                 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER;
4548
4549         netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 |
4550                 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
4551
4552         netdev->priv_flags |= IFF_UNICAST_FLT;
4553
4554         netdev->flags |= IFF_MULTICAST;
4555
4556         netif_set_gso_max_size(netdev, 65535 - ETH_HLEN);
4557
4558         netdev->netdev_ops = &be_netdev_ops;
4559
4560         netdev->ethtool_ops = &be_ethtool_ops;
4561 }
4562
4563 static void be_unmap_pci_bars(struct be_adapter *adapter)
4564 {
4565         if (adapter->csr)
4566                 pci_iounmap(adapter->pdev, adapter->csr);
4567         if (adapter->db)
4568                 pci_iounmap(adapter->pdev, adapter->db);
4569 }
4570
4571 static int db_bar(struct be_adapter *adapter)
4572 {
4573         if (lancer_chip(adapter) || !be_physfn(adapter))
4574                 return 0;
4575         else
4576                 return 4;
4577 }
4578
4579 static int be_roce_map_pci_bars(struct be_adapter *adapter)
4580 {
4581         if (skyhawk_chip(adapter)) {
4582                 adapter->roce_db.size = 4096;
4583                 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev,
4584                                                               db_bar(adapter));
4585                 adapter->roce_db.total_size = pci_resource_len(adapter->pdev,
4586                                                                db_bar(adapter));
4587         }
4588         return 0;
4589 }
4590
4591 static int be_map_pci_bars(struct be_adapter *adapter)
4592 {
4593         u8 __iomem *addr;
4594
4595         if (BEx_chip(adapter) && be_physfn(adapter)) {
4596                 adapter->csr = pci_iomap(adapter->pdev, 2, 0);
4597                 if (!adapter->csr)
4598                         return -ENOMEM;
4599         }
4600
4601         addr = pci_iomap(adapter->pdev, db_bar(adapter), 0);
4602         if (!addr)
4603                 goto pci_map_err;
4604         adapter->db = addr;
4605
4606         be_roce_map_pci_bars(adapter);
4607         return 0;
4608
4609 pci_map_err:
4610         dev_err(&adapter->pdev->dev, "Error in mapping PCI BARs\n");
4611         be_unmap_pci_bars(adapter);
4612         return -ENOMEM;
4613 }
4614
4615 static void be_ctrl_cleanup(struct be_adapter *adapter)
4616 {
4617         struct be_dma_mem *mem = &adapter->mbox_mem_alloced;
4618
4619         be_unmap_pci_bars(adapter);
4620
4621         if (mem->va)
4622                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4623                                   mem->dma);
4624
4625         mem = &adapter->rx_filter;
4626         if (mem->va)
4627                 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va,
4628                                   mem->dma);
4629 }
4630
4631 static int be_ctrl_init(struct be_adapter *adapter)
4632 {
4633         struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced;
4634         struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem;
4635         struct be_dma_mem *rx_filter = &adapter->rx_filter;
4636         u32 sli_intf;
4637         int status;
4638
4639         pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf);
4640         adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >>
4641                                  SLI_INTF_FAMILY_SHIFT;
4642         adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0;
4643
4644         status = be_map_pci_bars(adapter);
4645         if (status)
4646                 goto done;
4647
4648         mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16;
4649         mbox_mem_alloc->va = dma_alloc_coherent(&adapter->pdev->dev,
4650                                                 mbox_mem_alloc->size,
4651                                                 &mbox_mem_alloc->dma,
4652                                                 GFP_KERNEL);
4653         if (!mbox_mem_alloc->va) {
4654                 status = -ENOMEM;
4655                 goto unmap_pci_bars;
4656         }
4657         mbox_mem_align->size = sizeof(struct be_mcc_mailbox);
4658         mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16);
4659         mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16);
4660         memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox));
4661
4662         rx_filter->size = sizeof(struct be_cmd_req_rx_filter);
4663         rx_filter->va = dma_zalloc_coherent(&adapter->pdev->dev,
4664                                             rx_filter->size, &rx_filter->dma,
4665                                             GFP_KERNEL);
4666         if (!rx_filter->va) {
4667                 status = -ENOMEM;
4668                 goto free_mbox;
4669         }
4670
4671         mutex_init(&adapter->mbox_lock);
4672         spin_lock_init(&adapter->mcc_lock);
4673         spin_lock_init(&adapter->mcc_cq_lock);
4674
4675         init_completion(&adapter->et_cmd_compl);
4676         pci_save_state(adapter->pdev);
4677         return 0;
4678
4679 free_mbox:
4680         dma_free_coherent(&adapter->pdev->dev, mbox_mem_alloc->size,
4681                           mbox_mem_alloc->va, mbox_mem_alloc->dma);
4682
4683 unmap_pci_bars:
4684         be_unmap_pci_bars(adapter);
4685
4686 done:
4687         return status;
4688 }
4689
4690 static void be_stats_cleanup(struct be_adapter *adapter)
4691 {
4692         struct be_dma_mem *cmd = &adapter->stats_cmd;
4693
4694         if (cmd->va)
4695                 dma_free_coherent(&adapter->pdev->dev, cmd->size,
4696                                   cmd->va, cmd->dma);
4697 }
4698
4699 static int be_stats_init(struct be_adapter *adapter)
4700 {
4701         struct be_dma_mem *cmd = &adapter->stats_cmd;
4702
4703         if (lancer_chip(adapter))
4704                 cmd->size = sizeof(struct lancer_cmd_req_pport_stats);
4705         else if (BE2_chip(adapter))
4706                 cmd->size = sizeof(struct be_cmd_req_get_stats_v0);
4707         else if (BE3_chip(adapter))
4708                 cmd->size = sizeof(struct be_cmd_req_get_stats_v1);
4709         else
4710                 /* ALL non-BE ASICs */
4711                 cmd->size = sizeof(struct be_cmd_req_get_stats_v2);
4712
4713         cmd->va = dma_zalloc_coherent(&adapter->pdev->dev, cmd->size, &cmd->dma,
4714                                       GFP_KERNEL);
4715         if (!cmd->va)
4716                 return -ENOMEM;
4717         return 0;
4718 }
4719
4720 static void be_remove(struct pci_dev *pdev)
4721 {
4722         struct be_adapter *adapter = pci_get_drvdata(pdev);
4723
4724         if (!adapter)
4725                 return;
4726
4727         be_roce_dev_remove(adapter);
4728         be_intr_set(adapter, false);
4729
4730         cancel_delayed_work_sync(&adapter->func_recovery_work);
4731
4732         unregister_netdev(adapter->netdev);
4733
4734         be_clear(adapter);
4735
4736         /* tell fw we're done with firing cmds */
4737         be_cmd_fw_clean(adapter);
4738
4739         be_stats_cleanup(adapter);
4740
4741         be_ctrl_cleanup(adapter);
4742
4743         pci_disable_pcie_error_reporting(pdev);
4744
4745         pci_release_regions(pdev);
4746         pci_disable_device(pdev);
4747
4748         free_netdev(adapter->netdev);
4749 }
4750
4751 static int be_get_initial_config(struct be_adapter *adapter)
4752 {
4753         int status, level;
4754
4755         status = be_cmd_get_cntl_attributes(adapter);
4756         if (status)
4757                 return status;
4758
4759         /* Must be a power of 2 or else MODULO will BUG_ON */
4760         adapter->be_get_temp_freq = 64;
4761
4762         if (BEx_chip(adapter)) {
4763                 level = be_cmd_get_fw_log_level(adapter);
4764                 adapter->msg_enable =
4765                         level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0;
4766         }
4767
4768         adapter->cfg_num_qs = netif_get_num_default_rss_queues();
4769         return 0;
4770 }
4771
4772 static int lancer_recover_func(struct be_adapter *adapter)
4773 {
4774         struct device *dev = &adapter->pdev->dev;
4775         int status;
4776
4777         status = lancer_test_and_set_rdy_state(adapter);
4778         if (status)
4779                 goto err;
4780
4781         if (netif_running(adapter->netdev))
4782                 be_close(adapter->netdev);
4783
4784         be_clear(adapter);
4785
4786         be_clear_all_error(adapter);
4787
4788         status = be_setup(adapter);
4789         if (status)
4790                 goto err;
4791
4792         if (netif_running(adapter->netdev)) {
4793                 status = be_open(adapter->netdev);
4794                 if (status)
4795                         goto err;
4796         }
4797
4798         dev_err(dev, "Adapter recovery successful\n");
4799         return 0;
4800 err:
4801         if (status == -EAGAIN)
4802                 dev_err(dev, "Waiting for resource provisioning\n");
4803         else
4804                 dev_err(dev, "Adapter recovery failed\n");
4805
4806         return status;
4807 }
4808
4809 static void be_func_recovery_task(struct work_struct *work)
4810 {
4811         struct be_adapter *adapter =
4812                 container_of(work, struct be_adapter,  func_recovery_work.work);
4813         int status = 0;
4814
4815         be_detect_error(adapter);
4816
4817         if (adapter->hw_error && lancer_chip(adapter)) {
4818                 rtnl_lock();
4819                 netif_device_detach(adapter->netdev);
4820                 rtnl_unlock();
4821
4822                 status = lancer_recover_func(adapter);
4823                 if (!status)
4824                         netif_device_attach(adapter->netdev);
4825         }
4826
4827         /* In Lancer, for all errors other than provisioning error (-EAGAIN),
4828          * no need to attempt further recovery.
4829          */
4830         if (!status || status == -EAGAIN)
4831                 schedule_delayed_work(&adapter->func_recovery_work,
4832                                       msecs_to_jiffies(1000));
4833 }
4834
4835 static void be_worker(struct work_struct *work)
4836 {
4837         struct be_adapter *adapter =
4838                 container_of(work, struct be_adapter, work.work);
4839         struct be_rx_obj *rxo;
4840         int i;
4841
4842         /* when interrupts are not yet enabled, just reap any pending
4843         * mcc completions */
4844         if (!netif_running(adapter->netdev)) {
4845                 local_bh_disable();
4846                 be_process_mcc(adapter);
4847                 local_bh_enable();
4848                 goto reschedule;
4849         }
4850
4851         if (!adapter->stats_cmd_sent) {
4852                 if (lancer_chip(adapter))
4853                         lancer_cmd_get_pport_stats(adapter,
4854                                                    &adapter->stats_cmd);
4855                 else
4856                         be_cmd_get_stats(adapter, &adapter->stats_cmd);
4857         }
4858
4859         if (be_physfn(adapter) &&
4860             MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0)
4861                 be_cmd_get_die_temperature(adapter);
4862
4863         for_all_rx_queues(adapter, rxo, i) {
4864                 /* Replenish RX-queues starved due to memory
4865                  * allocation failures.
4866                  */
4867                 if (rxo->rx_post_starved)
4868                         be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST);
4869         }
4870
4871         be_eqd_update(adapter);
4872
4873 reschedule:
4874         adapter->work_counter++;
4875         schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000));
4876 }
4877
4878 /* If any VFs are already enabled don't FLR the PF */
4879 static bool be_reset_required(struct be_adapter *adapter)
4880 {
4881         return pci_num_vf(adapter->pdev) ? false : true;
4882 }
4883
4884 static char *mc_name(struct be_adapter *adapter)
4885 {
4886         char *str = ""; /* default */
4887
4888         switch (adapter->mc_type) {
4889         case UMC:
4890                 str = "UMC";
4891                 break;
4892         case FLEX10:
4893                 str = "FLEX10";
4894                 break;
4895         case vNIC1:
4896                 str = "vNIC-1";
4897                 break;
4898         case nPAR:
4899                 str = "nPAR";
4900                 break;
4901         case UFP:
4902                 str = "UFP";
4903                 break;
4904         case vNIC2:
4905                 str = "vNIC-2";
4906                 break;
4907         default:
4908                 str = "";
4909         }
4910
4911         return str;
4912 }
4913
4914 static inline char *func_name(struct be_adapter *adapter)
4915 {
4916         return be_physfn(adapter) ? "PF" : "VF";
4917 }
4918
4919 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id)
4920 {
4921         int status = 0;
4922         struct be_adapter *adapter;
4923         struct net_device *netdev;
4924         char port_name;
4925
4926         dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER);
4927
4928         status = pci_enable_device(pdev);
4929         if (status)
4930                 goto do_none;
4931
4932         status = pci_request_regions(pdev, DRV_NAME);
4933         if (status)
4934                 goto disable_dev;
4935         pci_set_master(pdev);
4936
4937         netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS);
4938         if (!netdev) {
4939                 status = -ENOMEM;
4940                 goto rel_reg;
4941         }
4942         adapter = netdev_priv(netdev);
4943         adapter->pdev = pdev;
4944         pci_set_drvdata(pdev, adapter);
4945         adapter->netdev = netdev;
4946         SET_NETDEV_DEV(netdev, &pdev->dev);
4947
4948         status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
4949         if (!status) {
4950                 netdev->features |= NETIF_F_HIGHDMA;
4951         } else {
4952                 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
4953                 if (status) {
4954                         dev_err(&pdev->dev, "Could not set PCI DMA Mask\n");
4955                         goto free_netdev;
4956                 }
4957         }
4958
4959         status = pci_enable_pcie_error_reporting(pdev);
4960         if (!status)
4961                 dev_info(&pdev->dev, "PCIe error reporting enabled\n");
4962
4963         status = be_ctrl_init(adapter);
4964         if (status)
4965                 goto free_netdev;
4966
4967         /* sync up with fw's ready state */
4968         if (be_physfn(adapter)) {
4969                 status = be_fw_wait_ready(adapter);
4970                 if (status)
4971                         goto ctrl_clean;
4972         }
4973
4974         if (be_reset_required(adapter)) {
4975                 status = be_cmd_reset_function(adapter);
4976                 if (status)
4977                         goto ctrl_clean;
4978
4979                 /* Wait for interrupts to quiesce after an FLR */
4980                 msleep(100);
4981         }
4982
4983         /* Allow interrupts for other ULPs running on NIC function */
4984         be_intr_set(adapter, true);
4985
4986         /* tell fw we're ready to fire cmds */
4987         status = be_cmd_fw_init(adapter);
4988         if (status)
4989                 goto ctrl_clean;
4990
4991         status = be_stats_init(adapter);
4992         if (status)
4993                 goto ctrl_clean;
4994
4995         status = be_get_initial_config(adapter);
4996         if (status)
4997                 goto stats_clean;
4998
4999         INIT_DELAYED_WORK(&adapter->work, be_worker);
5000         INIT_DELAYED_WORK(&adapter->func_recovery_work, be_func_recovery_task);
5001         adapter->rx_fc = true;
5002         adapter->tx_fc = true;
5003
5004         status = be_setup(adapter);
5005         if (status)
5006                 goto stats_clean;
5007
5008         be_netdev_init(netdev);
5009         status = register_netdev(netdev);
5010         if (status != 0)
5011                 goto unsetup;
5012
5013         be_roce_dev_add(adapter);
5014
5015         schedule_delayed_work(&adapter->func_recovery_work,
5016                               msecs_to_jiffies(1000));
5017
5018         be_cmd_query_port_name(adapter, &port_name);
5019
5020         dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev),
5021                  func_name(adapter), mc_name(adapter), port_name);
5022
5023         return 0;
5024
5025 unsetup:
5026         be_clear(adapter);
5027 stats_clean:
5028         be_stats_cleanup(adapter);
5029 ctrl_clean:
5030         be_ctrl_cleanup(adapter);
5031 free_netdev:
5032         free_netdev(netdev);
5033 rel_reg:
5034         pci_release_regions(pdev);
5035 disable_dev:
5036         pci_disable_device(pdev);
5037 do_none:
5038         dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev));
5039         return status;
5040 }
5041
5042 static int be_suspend(struct pci_dev *pdev, pm_message_t state)
5043 {
5044         struct be_adapter *adapter = pci_get_drvdata(pdev);
5045         struct net_device *netdev =  adapter->netdev;
5046
5047         if (adapter->wol_en)
5048                 be_setup_wol(adapter, true);
5049
5050         be_intr_set(adapter, false);
5051         cancel_delayed_work_sync(&adapter->func_recovery_work);
5052
5053         netif_device_detach(netdev);
5054         if (netif_running(netdev)) {
5055                 rtnl_lock();
5056                 be_close(netdev);
5057                 rtnl_unlock();
5058         }
5059         be_clear(adapter);
5060
5061         pci_save_state(pdev);
5062         pci_disable_device(pdev);
5063         pci_set_power_state(pdev, pci_choose_state(pdev, state));
5064         return 0;
5065 }
5066
5067 static int be_resume(struct pci_dev *pdev)
5068 {
5069         int status = 0;
5070         struct be_adapter *adapter = pci_get_drvdata(pdev);
5071         struct net_device *netdev =  adapter->netdev;
5072
5073         netif_device_detach(netdev);
5074
5075         status = pci_enable_device(pdev);
5076         if (status)
5077                 return status;
5078
5079         pci_set_power_state(pdev, PCI_D0);
5080         pci_restore_state(pdev);
5081
5082         status = be_fw_wait_ready(adapter);
5083         if (status)
5084                 return status;
5085
5086         be_intr_set(adapter, true);
5087         /* tell fw we're ready to fire cmds */
5088         status = be_cmd_fw_init(adapter);
5089         if (status)
5090                 return status;
5091
5092         be_setup(adapter);
5093         if (netif_running(netdev)) {
5094                 rtnl_lock();
5095                 be_open(netdev);
5096                 rtnl_unlock();
5097         }
5098
5099         schedule_delayed_work(&adapter->func_recovery_work,
5100                               msecs_to_jiffies(1000));
5101         netif_device_attach(netdev);
5102
5103         if (adapter->wol_en)
5104                 be_setup_wol(adapter, false);
5105
5106         return 0;
5107 }
5108
5109 /*
5110  * An FLR will stop BE from DMAing any data.
5111  */
5112 static void be_shutdown(struct pci_dev *pdev)
5113 {
5114         struct be_adapter *adapter = pci_get_drvdata(pdev);
5115
5116         if (!adapter)
5117                 return;
5118
5119         be_roce_dev_shutdown(adapter);
5120         cancel_delayed_work_sync(&adapter->work);
5121         cancel_delayed_work_sync(&adapter->func_recovery_work);
5122
5123         netif_device_detach(adapter->netdev);
5124
5125         be_cmd_reset_function(adapter);
5126
5127         pci_disable_device(pdev);
5128 }
5129
5130 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev,
5131                                             pci_channel_state_t state)
5132 {
5133         struct be_adapter *adapter = pci_get_drvdata(pdev);
5134         struct net_device *netdev =  adapter->netdev;
5135
5136         dev_err(&adapter->pdev->dev, "EEH error detected\n");
5137
5138         if (!adapter->eeh_error) {
5139                 adapter->eeh_error = true;
5140
5141                 cancel_delayed_work_sync(&adapter->func_recovery_work);
5142
5143                 rtnl_lock();
5144                 netif_device_detach(netdev);
5145                 if (netif_running(netdev))
5146                         be_close(netdev);
5147                 rtnl_unlock();
5148
5149                 be_clear(adapter);
5150         }
5151
5152         if (state == pci_channel_io_perm_failure)
5153                 return PCI_ERS_RESULT_DISCONNECT;
5154
5155         pci_disable_device(pdev);
5156
5157         /* The error could cause the FW to trigger a flash debug dump.
5158          * Resetting the card while flash dump is in progress
5159          * can cause it not to recover; wait for it to finish.
5160          * Wait only for first function as it is needed only once per
5161          * adapter.
5162          */
5163         if (pdev->devfn == 0)
5164                 ssleep(30);
5165
5166         return PCI_ERS_RESULT_NEED_RESET;
5167 }
5168
5169 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev)
5170 {
5171         struct be_adapter *adapter = pci_get_drvdata(pdev);
5172         int status;
5173
5174         dev_info(&adapter->pdev->dev, "EEH reset\n");
5175
5176         status = pci_enable_device(pdev);
5177         if (status)
5178                 return PCI_ERS_RESULT_DISCONNECT;
5179
5180         pci_set_master(pdev);
5181         pci_set_power_state(pdev, PCI_D0);
5182         pci_restore_state(pdev);
5183
5184         /* Check if card is ok and fw is ready */
5185         dev_info(&adapter->pdev->dev,
5186                  "Waiting for FW to be ready after EEH reset\n");
5187         status = be_fw_wait_ready(adapter);
5188         if (status)
5189                 return PCI_ERS_RESULT_DISCONNECT;
5190
5191         pci_cleanup_aer_uncorrect_error_status(pdev);
5192         be_clear_all_error(adapter);
5193         return PCI_ERS_RESULT_RECOVERED;
5194 }
5195
5196 static void be_eeh_resume(struct pci_dev *pdev)
5197 {
5198         int status = 0;
5199         struct be_adapter *adapter = pci_get_drvdata(pdev);
5200         struct net_device *netdev =  adapter->netdev;
5201
5202         dev_info(&adapter->pdev->dev, "EEH resume\n");
5203
5204         pci_save_state(pdev);
5205
5206         status = be_cmd_reset_function(adapter);
5207         if (status)
5208                 goto err;
5209
5210         /* On some BE3 FW versions, after a HW reset,
5211          * interrupts will remain disabled for each function.
5212          * So, explicitly enable interrupts
5213          */
5214         be_intr_set(adapter, true);
5215
5216         /* tell fw we're ready to fire cmds */
5217         status = be_cmd_fw_init(adapter);
5218         if (status)
5219                 goto err;
5220
5221         status = be_setup(adapter);
5222         if (status)
5223                 goto err;
5224
5225         if (netif_running(netdev)) {
5226                 status = be_open(netdev);
5227                 if (status)
5228                         goto err;
5229         }
5230
5231         schedule_delayed_work(&adapter->func_recovery_work,
5232                               msecs_to_jiffies(1000));
5233         netif_device_attach(netdev);
5234         return;
5235 err:
5236         dev_err(&adapter->pdev->dev, "EEH resume failed\n");
5237 }
5238
5239 static const struct pci_error_handlers be_eeh_handlers = {
5240         .error_detected = be_eeh_err_detected,
5241         .slot_reset = be_eeh_reset,
5242         .resume = be_eeh_resume,
5243 };
5244
5245 static struct pci_driver be_driver = {
5246         .name = DRV_NAME,
5247         .id_table = be_dev_ids,
5248         .probe = be_probe,
5249         .remove = be_remove,
5250         .suspend = be_suspend,
5251         .resume = be_resume,
5252         .shutdown = be_shutdown,
5253         .err_handler = &be_eeh_handlers
5254 };
5255
5256 static int __init be_init_module(void)
5257 {
5258         if (rx_frag_size != 8192 && rx_frag_size != 4096 &&
5259             rx_frag_size != 2048) {
5260                 printk(KERN_WARNING DRV_NAME
5261                         " : Module param rx_frag_size must be 2048/4096/8192."
5262                         " Using 2048\n");
5263                 rx_frag_size = 2048;
5264         }
5265
5266         return pci_register_driver(&be_driver);
5267 }
5268 module_init(be_init_module);
5269
5270 static void __exit be_exit_module(void)
5271 {
5272         pci_unregister_driver(&be_driver);
5273 }
5274 module_exit(be_exit_module);