drivers/infiniband/hw/mlx4/main.c

   1 /*
   2  * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved.
   3  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #include <linux/module.h>
  35 #include <linux/init.h>
  36 #include <linux/slab.h>
  37 #include <linux/errno.h>
  38 #include <linux/netdevice.h>
  39 #include <linux/inetdevice.h>
  40 #include <linux/rtnetlink.h>
  41 #include <linux/if_vlan.h>
  42
  43 #include <rdma/ib_smi.h>
  44 #include <rdma/ib_user_verbs.h>
  45 #include <rdma/ib_addr.h>
  46
  47 #include <linux/mlx4/driver.h>
  48 #include <linux/mlx4/cmd.h>
  49
  50 #include "mlx4_ib.h"
  51 #include "user.h"
  52
  53 #define DRV_NAME        MLX4_IB_DRV_NAME
  54 #define DRV_VERSION     "1.0"
  55 #define DRV_RELDATE     "April 4, 2008"
  56
  57 #define MLX4_IB_FLOW_MAX_PRIO 0xFFF
  58
  59 MODULE_AUTHOR("Roland Dreier");
  60 MODULE_DESCRIPTION("Mellanox ConnectX HCA InfiniBand driver");
  61 MODULE_LICENSE("Dual BSD/GPL");
  62 MODULE_VERSION(DRV_VERSION);
  63
  64 int mlx4_ib_sm_guid_assign = 1;
  65 module_param_named(sm_guid_assign, mlx4_ib_sm_guid_assign, int, 0444);
  66 MODULE_PARM_DESC(sm_guid_assign, "Enable SM alias_GUID assignment if sm_guid_assign > 0 (Default: 1)");
  67
  68 static const char mlx4_ib_version[] =
  69         DRV_NAME ": Mellanox ConnectX InfiniBand driver v"
  70         DRV_VERSION " (" DRV_RELDATE ")\n";
  71
  72 struct update_gid_work {
  73         struct work_struct      work;
  74         union ib_gid            gids[128];
  75         struct mlx4_ib_dev     *dev;
  76         int                     port;
  77 };
  78
  79 static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init);
  80
  81 static struct workqueue_struct *wq;
  82
  83 static void init_query_mad(struct ib_smp *mad)
  84 {
  85         mad->base_version  = 1;
  86         mad->mgmt_class    = IB_MGMT_CLASS_SUBN_LID_ROUTED;
  87         mad->class_version = 1;
  88         mad->method        = IB_MGMT_METHOD_GET;
  89 }
  90
  91 static union ib_gid zgid;
  92
  93 static int check_flow_steering_support(struct mlx4_dev *dev)
  94 {
  95         int ib_num_ports = 0;
  96         int i;
  97
  98         mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB)
  99                 ib_num_ports++;
 100
 101         if (dev->caps.steering_mode == MLX4_STEERING_MODE_DEVICE_MANAGED) {
 102                 if (ib_num_ports || mlx4_is_mfunc(dev)) {
 103                         pr_warn("Device managed flow steering is unavailable "
 104                                 "for IB ports or in multifunction env.\n");
 105                         return 0;
 106                 }
 107                 return 1;
 108         }
 109         return 0;
 110 }
 111
 112 static int mlx4_ib_query_device(struct ib_device *ibdev,
 113                                 struct ib_device_attr *props)
 114 {
 115         struct mlx4_ib_dev *dev = to_mdev(ibdev);
 116         struct ib_smp *in_mad  = NULL;
 117         struct ib_smp *out_mad = NULL;
 118         int err = -ENOMEM;
 119
 120         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
 121         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
 122         if (!in_mad || !out_mad)
 123                 goto out;
 124
 125         init_query_mad(in_mad);
 126         in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
 127
 128         err = mlx4_MAD_IFC(to_mdev(ibdev), MLX4_MAD_IFC_IGNORE_KEYS,
 129                            1, NULL, NULL, in_mad, out_mad);
 130         if (err)
 131                 goto out;
 132
 133         memset(props, 0, sizeof *props);
 134
 135         props->fw_ver = dev->dev->caps.fw_ver;
 136         props->device_cap_flags    = IB_DEVICE_CHANGE_PHY_PORT |
 137                 IB_DEVICE_PORT_ACTIVE_EVENT             |
 138                 IB_DEVICE_SYS_IMAGE_GUID                |
 139                 IB_DEVICE_RC_RNR_NAK_GEN                |
 140                 IB_DEVICE_BLOCK_MULTICAST_LOOPBACK;
 141         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_BAD_PKEY_CNTR)
 142                 props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
 143         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_BAD_QKEY_CNTR)
 144                 props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
 145         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_APM)
 146                 props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
 147         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_UD_AV_PORT)
 148                 props->device_cap_flags |= IB_DEVICE_UD_AV_PORT_ENFORCE;
 149         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_IPOIB_CSUM)
 150                 props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
 151         if (dev->dev->caps.max_gso_sz && dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_BLH)
 152                 props->device_cap_flags |= IB_DEVICE_UD_TSO;
 153         if (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_RESERVED_LKEY)
 154                 props->device_cap_flags |= IB_DEVICE_LOCAL_DMA_LKEY;
 155         if ((dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_LOCAL_INV) &&
 156             (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_REMOTE_INV) &&
 157             (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_FAST_REG_WR))
 158                 props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
 159         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC)
 160                 props->device_cap_flags |= IB_DEVICE_XRC;
 161         if (dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW)
 162                 props->device_cap_flags |= IB_DEVICE_MEM_WINDOW;
 163         if (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN) {
 164                 if (dev->dev->caps.bmme_flags & MLX4_BMME_FLAG_WIN_TYPE_2B)
 165                         props->device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2B;
 166                 else
 167                         props->device_cap_flags |= IB_DEVICE_MEM_WINDOW_TYPE_2A;
 168         if (check_flow_steering_support(dev->dev))
 169                 props->device_cap_flags |= IB_DEVICE_MANAGED_FLOW_STEERING;
 170         }
 171
 172         props->vendor_id           = be32_to_cpup((__be32 *) (out_mad->data + 36)) &
 173                 0xffffff;
 174         props->vendor_part_id      = dev->dev->pdev->device;
 175         props->hw_ver              = be32_to_cpup((__be32 *) (out_mad->data + 32));
 176         memcpy(&props->sys_image_guid, out_mad->data +  4, 8);
 177
 178         props->max_mr_size         = ~0ull;
 179         props->page_size_cap       = dev->dev->caps.page_size_cap;
 180         props->max_qp              = dev->dev->quotas.qp;
 181         props->max_qp_wr           = dev->dev->caps.max_wqes - MLX4_IB_SQ_MAX_SPARE;
 182         props->max_sge             = min(dev->dev->caps.max_sq_sg,
 183                                          dev->dev->caps.max_rq_sg);
 184         props->max_cq              = dev->dev->quotas.cq;
 185         props->max_cqe             = dev->dev->caps.max_cqes;
 186         props->max_mr              = dev->dev->quotas.mpt;
 187         props->max_pd              = dev->dev->caps.num_pds - dev->dev->caps.reserved_pds;
 188         props->max_qp_rd_atom      = dev->dev->caps.max_qp_dest_rdma;
 189         props->max_qp_init_rd_atom = dev->dev->caps.max_qp_init_rdma;
 190         props->max_res_rd_atom     = props->max_qp_rd_atom * props->max_qp;
 191         props->max_srq             = dev->dev->quotas.srq;
 192         props->max_srq_wr          = dev->dev->caps.max_srq_wqes - 1;
 193         props->max_srq_sge         = dev->dev->caps.max_srq_sge;
 194         props->max_fast_reg_page_list_len = MLX4_MAX_FAST_REG_PAGES;
 195         props->local_ca_ack_delay  = dev->dev->caps.local_ca_ack_delay;
 196         props->atomic_cap          = dev->dev->caps.flags & MLX4_DEV_CAP_FLAG_ATOMIC ?
 197                 IB_ATOMIC_HCA : IB_ATOMIC_NONE;
 198         props->masked_atomic_cap   = props->atomic_cap;
 199         props->max_pkeys           = dev->dev->caps.pkey_table_len[1];
 200         props->max_mcast_grp       = dev->dev->caps.num_mgms + dev->dev->caps.num_amgms;
 201         props->max_mcast_qp_attach = dev->dev->caps.num_qp_per_mgm;
 202         props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
 203                                            props->max_mcast_grp;
 204         props->max_map_per_fmr = dev->dev->caps.max_fmr_maps;
 205
 206 out:
 207         kfree(in_mad);
 208         kfree(out_mad);
 209
 210         return err;
 211 }
 212
 213 static enum rdma_link_layer
 214 mlx4_ib_port_link_layer(struct ib_device *device, u8 port_num)
 215 {
 216         struct mlx4_dev *dev = to_mdev(device)->dev;
 217
 218         return dev->caps.port_mask[port_num] == MLX4_PORT_TYPE_IB ?
 219                 IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET;
 220 }
 221
 222 static int ib_link_query_port(struct ib_device *ibdev, u8 port,
 223                               struct ib_port_attr *props, int netw_view)
 224 {
 225         struct ib_smp *in_mad  = NULL;
 226         struct ib_smp *out_mad = NULL;
 227         int ext_active_speed;
 228         int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS;
 229         int err = -ENOMEM;
 230
 231         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
 232         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
 233         if (!in_mad || !out_mad)
 234                 goto out;
 235
 236         init_query_mad(in_mad);
 237         in_mad->attr_id  = IB_SMP_ATTR_PORT_INFO;
 238         in_mad->attr_mod = cpu_to_be32(port);
 239
 240         if (mlx4_is_mfunc(to_mdev(ibdev)->dev) && netw_view)
 241                 mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW;
 242
 243         err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port, NULL, NULL,
 244                                 in_mad, out_mad);
 245         if (err)
 246                 goto out;
 247
 248
 249         props->lid              = be16_to_cpup((__be16 *) (out_mad->data + 16));
 250         props->lmc              = out_mad->data[34] & 0x7;
 251         props->sm_lid           = be16_to_cpup((__be16 *) (out_mad->data + 18));
 252         props->sm_sl            = out_mad->data[36] & 0xf;
 253         props->state            = out_mad->data[32] & 0xf;
 254         props->phys_state       = out_mad->data[33] >> 4;
 255         props->port_cap_flags   = be32_to_cpup((__be32 *) (out_mad->data + 20));
 256         if (netw_view)
 257                 props->gid_tbl_len = out_mad->data[50];
 258         else
 259                 props->gid_tbl_len = to_mdev(ibdev)->dev->caps.gid_table_len[port];
 260         props->max_msg_sz       = to_mdev(ibdev)->dev->caps.max_msg_sz;
 261         props->pkey_tbl_len     = to_mdev(ibdev)->dev->caps.pkey_table_len[port];
 262         props->bad_pkey_cntr    = be16_to_cpup((__be16 *) (out_mad->data + 46));
 263         props->qkey_viol_cntr   = be16_to_cpup((__be16 *) (out_mad->data + 48));
 264         props->active_width     = out_mad->data[31] & 0xf;
 265         props->active_speed     = out_mad->data[35] >> 4;
 266         props->max_mtu          = out_mad->data[41] & 0xf;
 267         props->active_mtu       = out_mad->data[36] >> 4;
 268         props->subnet_timeout   = out_mad->data[51] & 0x1f;
 269         props->max_vl_num       = out_mad->data[37] >> 4;
 270         props->init_type_reply  = out_mad->data[41] >> 4;
 271
 272         /* Check if extended speeds (EDR/FDR/...) are supported */
 273         if (props->port_cap_flags & IB_PORT_EXTENDED_SPEEDS_SUP) {
 274                 ext_active_speed = out_mad->data[62] >> 4;
 275
 276                 switch (ext_active_speed) {
 277                 case 1:
 278                         props->active_speed = IB_SPEED_FDR;
 279                         break;
 280                 case 2:
 281                         props->active_speed = IB_SPEED_EDR;
 282                         break;
 283                 }
 284         }
 285
 286         /* If reported active speed is QDR, check if is FDR-10 */
 287         if (props->active_speed == IB_SPEED_QDR) {
 288                 init_query_mad(in_mad);
 289                 in_mad->attr_id = MLX4_ATTR_EXTENDED_PORT_INFO;
 290                 in_mad->attr_mod = cpu_to_be32(port);
 291
 292                 err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port,
 293                                    NULL, NULL, in_mad, out_mad);
 294                 if (err)
 295                         goto out;
 296
 297                 /* Checking LinkSpeedActive for FDR-10 */
 298                 if (out_mad->data[15] & 0x1)
 299                         props->active_speed = IB_SPEED_FDR10;
 300         }
 301
 302         /* Avoid wrong speed value returned by FW if the IB link is down. */
 303         if (props->state == IB_PORT_DOWN)
 304                  props->active_speed = IB_SPEED_SDR;
 305
 306 out:
 307         kfree(in_mad);
 308         kfree(out_mad);
 309         return err;
 310 }
 311
 312 static u8 state_to_phys_state(enum ib_port_state state)
 313 {
 314         return state == IB_PORT_ACTIVE ? 5 : 3;
 315 }
 316
 317 static int eth_link_query_port(struct ib_device *ibdev, u8 port,
 318                                struct ib_port_attr *props, int netw_view)
 319 {
 320
 321         struct mlx4_ib_dev *mdev = to_mdev(ibdev);
 322         struct mlx4_ib_iboe *iboe = &mdev->iboe;
 323         struct net_device *ndev;
 324         enum ib_mtu tmp;
 325         struct mlx4_cmd_mailbox *mailbox;
 326         int err = 0;
 327
 328         mailbox = mlx4_alloc_cmd_mailbox(mdev->dev);
 329         if (IS_ERR(mailbox))
 330                 return PTR_ERR(mailbox);
 331
 332         err = mlx4_cmd_box(mdev->dev, 0, mailbox->dma, port, 0,
 333                            MLX4_CMD_QUERY_PORT, MLX4_CMD_TIME_CLASS_B,
 334                            MLX4_CMD_WRAPPED);
 335         if (err)
 336                 goto out;
 337
 338         props->active_width     =  (((u8 *)mailbox->buf)[5] == 0x40) ?
 339                                                 IB_WIDTH_4X : IB_WIDTH_1X;
 340         props->active_speed     = IB_SPEED_QDR;
 341         props->port_cap_flags   = IB_PORT_CM_SUP;
 342         props->gid_tbl_len      = mdev->dev->caps.gid_table_len[port];
 343         props->max_msg_sz       = mdev->dev->caps.max_msg_sz;
 344         props->pkey_tbl_len     = 1;
 345         props->max_mtu          = IB_MTU_4096;
 346         props->max_vl_num       = 2;
 347         props->state            = IB_PORT_DOWN;
 348         props->phys_state       = state_to_phys_state(props->state);
 349         props->active_mtu       = IB_MTU_256;
 350         spin_lock(&iboe->lock);
 351         ndev = iboe->netdevs[port - 1];
 352         if (!ndev)
 353                 goto out_unlock;
 354
 355         tmp = iboe_get_mtu(ndev->mtu);
 356         props->active_mtu = tmp ? min(props->max_mtu, tmp) : IB_MTU_256;
 357
 358         props->state            = (netif_running(ndev) && netif_carrier_ok(ndev)) ?
 359                                         IB_PORT_ACTIVE : IB_PORT_DOWN;
 360         props->phys_state       = state_to_phys_state(props->state);
 361 out_unlock:
 362         spin_unlock(&iboe->lock);
 363 out:
 364         mlx4_free_cmd_mailbox(mdev->dev, mailbox);
 365         return err;
 366 }
 367
 368 int __mlx4_ib_query_port(struct ib_device *ibdev, u8 port,
 369                          struct ib_port_attr *props, int netw_view)
 370 {
 371         int err;
 372
 373         memset(props, 0, sizeof *props);
 374
 375         err = mlx4_ib_port_link_layer(ibdev, port) == IB_LINK_LAYER_INFINIBAND ?
 376                 ib_link_query_port(ibdev, port, props, netw_view) :
 377                                 eth_link_query_port(ibdev, port, props, netw_view);
 378
 379         return err;
 380 }
 381
 382 static int mlx4_ib_query_port(struct ib_device *ibdev, u8 port,
 383                               struct ib_port_attr *props)
 384 {
 385         /* returns host view */
 386         return __mlx4_ib_query_port(ibdev, port, props, 0);
 387 }
 388
 389 int __mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
 390                         union ib_gid *gid, int netw_view)
 391 {
 392         struct ib_smp *in_mad  = NULL;
 393         struct ib_smp *out_mad = NULL;
 394         int err = -ENOMEM;
 395         struct mlx4_ib_dev *dev = to_mdev(ibdev);
 396         int clear = 0;
 397         int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS;
 398
 399         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
 400         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
 401         if (!in_mad || !out_mad)
 402                 goto out;
 403
 404         init_query_mad(in_mad);
 405         in_mad->attr_id  = IB_SMP_ATTR_PORT_INFO;
 406         in_mad->attr_mod = cpu_to_be32(port);
 407
 408         if (mlx4_is_mfunc(dev->dev) && netw_view)
 409                 mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW;
 410
 411         err = mlx4_MAD_IFC(dev, mad_ifc_flags, port, NULL, NULL, in_mad, out_mad);
 412         if (err)
 413                 goto out;
 414
 415         memcpy(gid->raw, out_mad->data + 8, 8);
 416
 417         if (mlx4_is_mfunc(dev->dev) && !netw_view) {
 418                 if (index) {
 419                         /* For any index > 0, return the null guid */
 420                         err = 0;
 421                         clear = 1;
 422                         goto out;
 423                 }
 424         }
 425
 426         init_query_mad(in_mad);
 427         in_mad->attr_id  = IB_SMP_ATTR_GUID_INFO;
 428         in_mad->attr_mod = cpu_to_be32(index / 8);
 429
 430         err = mlx4_MAD_IFC(dev, mad_ifc_flags, port,
 431                            NULL, NULL, in_mad, out_mad);
 432         if (err)
 433                 goto out;
 434
 435         memcpy(gid->raw + 8, out_mad->data + (index % 8) * 8, 8);
 436
 437 out:
 438         if (clear)
 439                 memset(gid->raw + 8, 0, 8);
 440         kfree(in_mad);
 441         kfree(out_mad);
 442         return err;
 443 }
 444
 445 static int iboe_query_gid(struct ib_device *ibdev, u8 port, int index,
 446                           union ib_gid *gid)
 447 {
 448         struct mlx4_ib_dev *dev = to_mdev(ibdev);
 449
 450         *gid = dev->iboe.gid_table[port - 1][index];
 451
 452         return 0;
 453 }
 454
 455 static int mlx4_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
 456                              union ib_gid *gid)
 457 {
 458         if (rdma_port_get_link_layer(ibdev, port) == IB_LINK_LAYER_INFINIBAND)
 459                 return __mlx4_ib_query_gid(ibdev, port, index, gid, 0);
 460         else
 461                 return iboe_query_gid(ibdev, port, index, gid);
 462 }
 463
 464 int __mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
 465                          u16 *pkey, int netw_view)
 466 {
 467         struct ib_smp *in_mad  = NULL;
 468         struct ib_smp *out_mad = NULL;
 469         int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS;
 470         int err = -ENOMEM;
 471
 472         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
 473         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
 474         if (!in_mad || !out_mad)
 475                 goto out;
 476
 477         init_query_mad(in_mad);
 478         in_mad->attr_id  = IB_SMP_ATTR_PKEY_TABLE;
 479         in_mad->attr_mod = cpu_to_be32(index / 32);
 480
 481         if (mlx4_is_mfunc(to_mdev(ibdev)->dev) && netw_view)
 482                 mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW;
 483
 484         err = mlx4_MAD_IFC(to_mdev(ibdev), mad_ifc_flags, port, NULL, NULL,
 485                            in_mad, out_mad);
 486         if (err)
 487                 goto out;
 488
 489         *pkey = be16_to_cpu(((__be16 *) out_mad->data)[index % 32]);
 490
 491 out:
 492         kfree(in_mad);
 493         kfree(out_mad);
 494         return err;
 495 }
 496
 497 static int mlx4_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
 498 {
 499         return __mlx4_ib_query_pkey(ibdev, port, index, pkey, 0);
 500 }
 501
 502 static int mlx4_ib_modify_device(struct ib_device *ibdev, int mask,
 503                                  struct ib_device_modify *props)
 504 {
 505         struct mlx4_cmd_mailbox *mailbox;
 506         unsigned long flags;
 507
 508         if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
 509                 return -EOPNOTSUPP;
 510
 511         if (!(mask & IB_DEVICE_MODIFY_NODE_DESC))
 512                 return 0;
 513
 514         if (mlx4_is_slave(to_mdev(ibdev)->dev))
 515                 return -EOPNOTSUPP;
 516
 517         spin_lock_irqsave(&to_mdev(ibdev)->sm_lock, flags);
 518         memcpy(ibdev->node_desc, props->node_desc, 64);
 519         spin_unlock_irqrestore(&to_mdev(ibdev)->sm_lock, flags);
 520
 521         /*
 522          * If possible, pass node desc to FW, so it can generate
 523          * a 144 trap.  If cmd fails, just ignore.
 524          */
 525         mailbox = mlx4_alloc_cmd_mailbox(to_mdev(ibdev)->dev);
 526         if (IS_ERR(mailbox))
 527                 return 0;
 528
 529         memcpy(mailbox->buf, props->node_desc, 64);
 530         mlx4_cmd(to_mdev(ibdev)->dev, mailbox->dma, 1, 0,
 531                  MLX4_CMD_SET_NODE, MLX4_CMD_TIME_CLASS_A, MLX4_CMD_NATIVE);
 532
 533         mlx4_free_cmd_mailbox(to_mdev(ibdev)->dev, mailbox);
 534
 535         return 0;
 536 }
 537
 538 static int mlx4_SET_PORT(struct mlx4_ib_dev *dev, u8 port, int reset_qkey_viols,
 539                          u32 cap_mask)
 540 {
 541         struct mlx4_cmd_mailbox *mailbox;
 542         int err;
 543         u8 is_eth = dev->dev->caps.port_type[port] == MLX4_PORT_TYPE_ETH;
 544
 545         mailbox = mlx4_alloc_cmd_mailbox(dev->dev);
 546         if (IS_ERR(mailbox))
 547                 return PTR_ERR(mailbox);
 548
 549         if (dev->dev->flags & MLX4_FLAG_OLD_PORT_CMDS) {
 550                 *(u8 *) mailbox->buf         = !!reset_qkey_viols << 6;
 551                 ((__be32 *) mailbox->buf)[2] = cpu_to_be32(cap_mask);
 552         } else {
 553                 ((u8 *) mailbox->buf)[3]     = !!reset_qkey_viols;
 554                 ((__be32 *) mailbox->buf)[1] = cpu_to_be32(cap_mask);
 555         }
 556
 557         err = mlx4_cmd(dev->dev, mailbox->dma, port, is_eth, MLX4_CMD_SET_PORT,
 558                        MLX4_CMD_TIME_CLASS_B, MLX4_CMD_NATIVE);
 559
 560         mlx4_free_cmd_mailbox(dev->dev, mailbox);
 561         return err;
 562 }
 563
 564 static int mlx4_ib_modify_port(struct ib_device *ibdev, u8 port, int mask,
 565                                struct ib_port_modify *props)
 566 {
 567         struct ib_port_attr attr;
 568         u32 cap_mask;
 569         int err;
 570
 571         mutex_lock(&to_mdev(ibdev)->cap_mask_mutex);
 572
 573         err = mlx4_ib_query_port(ibdev, port, &attr);
 574         if (err)
 575                 goto out;
 576
 577         cap_mask = (attr.port_cap_flags | props->set_port_cap_mask) &
 578                 ~props->clr_port_cap_mask;
 579
 580         err = mlx4_SET_PORT(to_mdev(ibdev), port,
 581                             !!(mask & IB_PORT_RESET_QKEY_CNTR),
 582                             cap_mask);
 583
 584 out:
 585         mutex_unlock(&to_mdev(ibdev)->cap_mask_mutex);
 586         return err;
 587 }
 588
 589 static struct ib_ucontext *mlx4_ib_alloc_ucontext(struct ib_device *ibdev,
 590                                                   struct ib_udata *udata)
 591 {
 592         struct mlx4_ib_dev *dev = to_mdev(ibdev);
 593         struct mlx4_ib_ucontext *context;
 594         struct mlx4_ib_alloc_ucontext_resp_v3 resp_v3;
 595         struct mlx4_ib_alloc_ucontext_resp resp;
 596         int err;
 597
 598         if (!dev->ib_active)
 599                 return ERR_PTR(-EAGAIN);
 600
 601         if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) {
 602                 resp_v3.qp_tab_size      = dev->dev->caps.num_qps;
 603                 resp_v3.bf_reg_size      = dev->dev->caps.bf_reg_size;
 604                 resp_v3.bf_regs_per_page = dev->dev->caps.bf_regs_per_page;
 605         } else {
 606                 resp.dev_caps         = dev->dev->caps.userspace_caps;
 607                 resp.qp_tab_size      = dev->dev->caps.num_qps;
 608                 resp.bf_reg_size      = dev->dev->caps.bf_reg_size;
 609                 resp.bf_regs_per_page = dev->dev->caps.bf_regs_per_page;
 610                 resp.cqe_size         = dev->dev->caps.cqe_size;
 611         }
 612
 613         context = kmalloc(sizeof *context, GFP_KERNEL);
 614         if (!context)
 615                 return ERR_PTR(-ENOMEM);
 616
 617         err = mlx4_uar_alloc(to_mdev(ibdev)->dev, &context->uar);
 618         if (err) {
 619                 kfree(context);
 620                 return ERR_PTR(err);
 621         }
 622
 623         INIT_LIST_HEAD(&context->db_page_list);
 624         mutex_init(&context->db_page_mutex);
 625
 626         if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION)
 627                 err = ib_copy_to_udata(udata, &resp_v3, sizeof(resp_v3));
 628         else
 629                 err = ib_copy_to_udata(udata, &resp, sizeof(resp));
 630
 631         if (err) {
 632                 mlx4_uar_free(to_mdev(ibdev)->dev, &context->uar);
 633                 kfree(context);
 634                 return ERR_PTR(-EFAULT);
 635         }
 636
 637         return &context->ibucontext;
 638 }
 639
 640 static int mlx4_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
 641 {
 642         struct mlx4_ib_ucontext *context = to_mucontext(ibcontext);
 643
 644         mlx4_uar_free(to_mdev(ibcontext->device)->dev, &context->uar);
 645         kfree(context);
 646
 647         return 0;
 648 }
 649
 650 static int mlx4_ib_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 651 {
 652         struct mlx4_ib_dev *dev = to_mdev(context->device);
 653
 654         if (vma->vm_end - vma->vm_start != PAGE_SIZE)
 655                 return -EINVAL;
 656
 657         if (vma->vm_pgoff == 0) {
 658                 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 659
 660                 if (io_remap_pfn_range(vma, vma->vm_start,
 661                                        to_mucontext(context)->uar.pfn,
 662                                        PAGE_SIZE, vma->vm_page_prot))
 663                         return -EAGAIN;
 664         } else if (vma->vm_pgoff == 1 && dev->dev->caps.bf_reg_size != 0) {
 665                 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 666
 667                 if (io_remap_pfn_range(vma, vma->vm_start,
 668                                        to_mucontext(context)->uar.pfn +
 669                                        dev->dev->caps.num_uars,
 670                                        PAGE_SIZE, vma->vm_page_prot))
 671                         return -EAGAIN;
 672         } else
 673                 return -EINVAL;
 674
 675         return 0;
 676 }
 677
 678 static struct ib_pd *mlx4_ib_alloc_pd(struct ib_device *ibdev,
 679                                       struct ib_ucontext *context,
 680                                       struct ib_udata *udata)
 681 {
 682         struct mlx4_ib_pd *pd;
 683         int err;
 684
 685         pd = kmalloc(sizeof *pd, GFP_KERNEL);
 686         if (!pd)
 687                 return ERR_PTR(-ENOMEM);
 688
 689         err = mlx4_pd_alloc(to_mdev(ibdev)->dev, &pd->pdn);
 690         if (err) {
 691                 kfree(pd);
 692                 return ERR_PTR(err);
 693         }
 694
 695         if (context)
 696                 if (ib_copy_to_udata(udata, &pd->pdn, sizeof (__u32))) {
 697                         mlx4_pd_free(to_mdev(ibdev)->dev, pd->pdn);
 698                         kfree(pd);
 699                         return ERR_PTR(-EFAULT);
 700                 }
 701
 702         return &pd->ibpd;
 703 }
 704
 705 static int mlx4_ib_dealloc_pd(struct ib_pd *pd)
 706 {
 707         mlx4_pd_free(to_mdev(pd->device)->dev, to_mpd(pd)->pdn);
 708         kfree(pd);
 709
 710         return 0;
 711 }
 712
 713 static struct ib_xrcd *mlx4_ib_alloc_xrcd(struct ib_device *ibdev,
 714                                           struct ib_ucontext *context,
 715                                           struct ib_udata *udata)
 716 {
 717         struct mlx4_ib_xrcd *xrcd;
 718         int err;
 719
 720         if (!(to_mdev(ibdev)->dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC))
 721                 return ERR_PTR(-ENOSYS);
 722
 723         xrcd = kmalloc(sizeof *xrcd, GFP_KERNEL);
 724         if (!xrcd)
 725                 return ERR_PTR(-ENOMEM);
 726
 727         err = mlx4_xrcd_alloc(to_mdev(ibdev)->dev, &xrcd->xrcdn);
 728         if (err)
 729                 goto err1;
 730
 731         xrcd->pd = ib_alloc_pd(ibdev);
 732         if (IS_ERR(xrcd->pd)) {
 733                 err = PTR_ERR(xrcd->pd);
 734                 goto err2;
 735         }
 736
 737         xrcd->cq = ib_create_cq(ibdev, NULL, NULL, xrcd, 1, 0);
 738         if (IS_ERR(xrcd->cq)) {
 739                 err = PTR_ERR(xrcd->cq);
 740                 goto err3;
 741         }
 742
 743         return &xrcd->ibxrcd;
 744
 745 err3:
 746         ib_dealloc_pd(xrcd->pd);
 747 err2:
 748         mlx4_xrcd_free(to_mdev(ibdev)->dev, xrcd->xrcdn);
 749 err1:
 750         kfree(xrcd);
 751         return ERR_PTR(err);
 752 }
 753
 754 static int mlx4_ib_dealloc_xrcd(struct ib_xrcd *xrcd)
 755 {
 756         ib_destroy_cq(to_mxrcd(xrcd)->cq);
 757         ib_dealloc_pd(to_mxrcd(xrcd)->pd);
 758         mlx4_xrcd_free(to_mdev(xrcd->device)->dev, to_mxrcd(xrcd)->xrcdn);
 759         kfree(xrcd);
 760
 761         return 0;
 762 }
 763
 764 static int add_gid_entry(struct ib_qp *ibqp, union ib_gid *gid)
 765 {
 766         struct mlx4_ib_qp *mqp = to_mqp(ibqp);
 767         struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
 768         struct mlx4_ib_gid_entry *ge;
 769
 770         ge = kzalloc(sizeof *ge, GFP_KERNEL);
 771         if (!ge)
 772                 return -ENOMEM;
 773
 774         ge->gid = *gid;
 775         if (mlx4_ib_add_mc(mdev, mqp, gid)) {
 776                 ge->port = mqp->port;
 777                 ge->added = 1;
 778         }
 779
 780         mutex_lock(&mqp->mutex);
 781         list_add_tail(&ge->list, &mqp->gid_list);
 782         mutex_unlock(&mqp->mutex);
 783
 784         return 0;
 785 }
 786
 787 int mlx4_ib_add_mc(struct mlx4_ib_dev *mdev, struct mlx4_ib_qp *mqp,
 788                    union ib_gid *gid)
 789 {
 790         u8 mac[6];
 791         struct net_device *ndev;
 792         int ret = 0;
 793
 794         if (!mqp->port)
 795                 return 0;
 796
 797         spin_lock(&mdev->iboe.lock);
 798         ndev = mdev->iboe.netdevs[mqp->port - 1];
 799         if (ndev)
 800                 dev_hold(ndev);
 801         spin_unlock(&mdev->iboe.lock);
 802
 803         if (ndev) {
 804                 rdma_get_mcast_mac((struct in6_addr *)gid, mac);
 805                 rtnl_lock();
 806                 dev_mc_add(mdev->iboe.netdevs[mqp->port - 1], mac);
 807                 ret = 1;
 808                 rtnl_unlock();
 809                 dev_put(ndev);
 810         }
 811
 812         return ret;
 813 }
 814
 815 struct mlx4_ib_steering {
 816         struct list_head list;
 817         u64 reg_id;
 818         union ib_gid gid;
 819 };
 820
 821 static int parse_flow_attr(struct mlx4_dev *dev,
 822                            union ib_flow_spec *ib_spec,
 823                            struct _rule_hw *mlx4_spec)
 824 {
 825         enum mlx4_net_trans_rule_id type;
 826
 827         switch (ib_spec->type) {
 828         case IB_FLOW_SPEC_ETH:
 829                 type = MLX4_NET_TRANS_RULE_ID_ETH;
 830                 memcpy(mlx4_spec->eth.dst_mac, ib_spec->eth.val.dst_mac,
 831                        ETH_ALEN);
 832                 memcpy(mlx4_spec->eth.dst_mac_msk, ib_spec->eth.mask.dst_mac,
 833                        ETH_ALEN);
 834                 mlx4_spec->eth.vlan_tag = ib_spec->eth.val.vlan_tag;
 835                 mlx4_spec->eth.vlan_tag_msk = ib_spec->eth.mask.vlan_tag;
 836                 break;
 837
 838         case IB_FLOW_SPEC_IPV4:
 839                 type = MLX4_NET_TRANS_RULE_ID_IPV4;
 840                 mlx4_spec->ipv4.src_ip = ib_spec->ipv4.val.src_ip;
 841                 mlx4_spec->ipv4.src_ip_msk = ib_spec->ipv4.mask.src_ip;
 842                 mlx4_spec->ipv4.dst_ip = ib_spec->ipv4.val.dst_ip;
 843                 mlx4_spec->ipv4.dst_ip_msk = ib_spec->ipv4.mask.dst_ip;
 844                 break;
 845
 846         case IB_FLOW_SPEC_TCP:
 847         case IB_FLOW_SPEC_UDP:
 848                 type = ib_spec->type == IB_FLOW_SPEC_TCP ?
 849                                         MLX4_NET_TRANS_RULE_ID_TCP :
 850                                         MLX4_NET_TRANS_RULE_ID_UDP;
 851                 mlx4_spec->tcp_udp.dst_port = ib_spec->tcp_udp.val.dst_port;
 852                 mlx4_spec->tcp_udp.dst_port_msk = ib_spec->tcp_udp.mask.dst_port;
 853                 mlx4_spec->tcp_udp.src_port = ib_spec->tcp_udp.val.src_port;
 854                 mlx4_spec->tcp_udp.src_port_msk = ib_spec->tcp_udp.mask.src_port;
 855                 break;
 856
 857         default:
 858                 return -EINVAL;
 859         }
 860         if (mlx4_map_sw_to_hw_steering_id(dev, type) < 0 ||
 861             mlx4_hw_rule_sz(dev, type) < 0)
 862                 return -EINVAL;
 863         mlx4_spec->id = cpu_to_be16(mlx4_map_sw_to_hw_steering_id(dev, type));
 864         mlx4_spec->size = mlx4_hw_rule_sz(dev, type) >> 2;
 865         return mlx4_hw_rule_sz(dev, type);
 866 }
 867
 868 static int __mlx4_ib_create_flow(struct ib_qp *qp, struct ib_flow_attr *flow_attr,
 869                           int domain,
 870                           enum mlx4_net_trans_promisc_mode flow_type,
 871                           u64 *reg_id)
 872 {
 873         int ret, i;
 874         int size = 0;
 875         void *ib_flow;
 876         struct mlx4_ib_dev *mdev = to_mdev(qp->device);
 877         struct mlx4_cmd_mailbox *mailbox;
 878         struct mlx4_net_trans_rule_hw_ctrl *ctrl;
 879
 880         static const u16 __mlx4_domain[] = {
 881                 [IB_FLOW_DOMAIN_USER] = MLX4_DOMAIN_UVERBS,
 882                 [IB_FLOW_DOMAIN_ETHTOOL] = MLX4_DOMAIN_ETHTOOL,
 883                 [IB_FLOW_DOMAIN_RFS] = MLX4_DOMAIN_RFS,
 884                 [IB_FLOW_DOMAIN_NIC] = MLX4_DOMAIN_NIC,
 885         };
 886
 887         if (flow_attr->priority > MLX4_IB_FLOW_MAX_PRIO) {
 888                 pr_err("Invalid priority value %d\n", flow_attr->priority);
 889                 return -EINVAL;
 890         }
 891
 892         if (domain >= IB_FLOW_DOMAIN_NUM) {
 893                 pr_err("Invalid domain value %d\n", domain);
 894                 return -EINVAL;
 895         }
 896
 897         if (mlx4_map_sw_to_hw_steering_mode(mdev->dev, flow_type) < 0)
 898                 return -EINVAL;
 899
 900         mailbox = mlx4_alloc_cmd_mailbox(mdev->dev);
 901         if (IS_ERR(mailbox))
 902                 return PTR_ERR(mailbox);
 903         ctrl = mailbox->buf;
 904
 905         ctrl->prio = cpu_to_be16(__mlx4_domain[domain] |
 906                                  flow_attr->priority);
 907         ctrl->type = mlx4_map_sw_to_hw_steering_mode(mdev->dev, flow_type);
 908         ctrl->port = flow_attr->port;
 909         ctrl->qpn = cpu_to_be32(qp->qp_num);
 910
 911         ib_flow = flow_attr + 1;
 912         size += sizeof(struct mlx4_net_trans_rule_hw_ctrl);
 913         for (i = 0; i < flow_attr->num_of_specs; i++) {
 914                 ret = parse_flow_attr(mdev->dev, ib_flow, mailbox->buf + size);
 915                 if (ret < 0) {
 916                         mlx4_free_cmd_mailbox(mdev->dev, mailbox);
 917                         return -EINVAL;
 918                 }
 919                 ib_flow += ((union ib_flow_spec *) ib_flow)->size;
 920                 size += ret;
 921         }
 922
 923         ret = mlx4_cmd_imm(mdev->dev, mailbox->dma, reg_id, size >> 2, 0,
 924                            MLX4_QP_FLOW_STEERING_ATTACH, MLX4_CMD_TIME_CLASS_A,
 925                            MLX4_CMD_NATIVE);
 926         if (ret == -ENOMEM)
 927                 pr_err("mcg table is full. Fail to register network rule.\n");
 928         else if (ret == -ENXIO)
 929                 pr_err("Device managed flow steering is disabled. Fail to register network rule.\n");
 930         else if (ret)
 931                 pr_err("Invalid argumant. Fail to register network rule.\n");
 932
 933         mlx4_free_cmd_mailbox(mdev->dev, mailbox);
 934         return ret;
 935 }
 936
 937 static int __mlx4_ib_destroy_flow(struct mlx4_dev *dev, u64 reg_id)
 938 {
 939         int err;
 940         err = mlx4_cmd(dev, reg_id, 0, 0,
 941                        MLX4_QP_FLOW_STEERING_DETACH, MLX4_CMD_TIME_CLASS_A,
 942                        MLX4_CMD_NATIVE);
 943         if (err)
 944                 pr_err("Fail to detach network rule. registration id = 0x%llx\n",
 945                        reg_id);
 946         return err;
 947 }
 948
 949 static struct ib_flow *mlx4_ib_create_flow(struct ib_qp *qp,
 950                                     struct ib_flow_attr *flow_attr,
 951                                     int domain)
 952 {
 953         int err = 0, i = 0;
 954         struct mlx4_ib_flow *mflow;
 955         enum mlx4_net_trans_promisc_mode type[2];
 956
 957         memset(type, 0, sizeof(type));
 958
 959         mflow = kzalloc(sizeof(*mflow), GFP_KERNEL);
 960         if (!mflow) {
 961                 err = -ENOMEM;
 962                 goto err_free;
 963         }
 964
 965         switch (flow_attr->type) {
 966         case IB_FLOW_ATTR_NORMAL:
 967                 type[0] = MLX4_FS_REGULAR;
 968                 break;
 969
 970         case IB_FLOW_ATTR_ALL_DEFAULT:
 971                 type[0] = MLX4_FS_ALL_DEFAULT;
 972                 break;
 973
 974         case IB_FLOW_ATTR_MC_DEFAULT:
 975                 type[0] = MLX4_FS_MC_DEFAULT;
 976                 break;
 977
 978         case IB_FLOW_ATTR_SNIFFER:
 979                 type[0] = MLX4_FS_UC_SNIFFER;
 980                 type[1] = MLX4_FS_MC_SNIFFER;
 981                 break;
 982
 983         default:
 984                 err = -EINVAL;
 985                 goto err_free;
 986         }
 987
 988         while (i < ARRAY_SIZE(type) && type[i]) {
 989                 err = __mlx4_ib_create_flow(qp, flow_attr, domain, type[i],
 990                                             &mflow->reg_id[i]);
 991                 if (err)
 992                         goto err_free;
 993                 i++;
 994         }
 995
 996         return &mflow->ibflow;
 997
 998 err_free:
 999         kfree(mflow);
1000         return ERR_PTR(err);
1001 }
1002
1003 static int mlx4_ib_destroy_flow(struct ib_flow *flow_id)
1004 {
1005         int err, ret = 0;
1006         int i = 0;
1007         struct mlx4_ib_dev *mdev = to_mdev(flow_id->qp->device);
1008         struct mlx4_ib_flow *mflow = to_mflow(flow_id);
1009
1010         while (i < ARRAY_SIZE(mflow->reg_id) && mflow->reg_id[i]) {
1011                 err = __mlx4_ib_destroy_flow(mdev->dev, mflow->reg_id[i]);
1012                 if (err)
1013                         ret = err;
1014                 i++;
1015         }
1016
1017         kfree(mflow);
1018         return ret;
1019 }
1020
1021 static int mlx4_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
1022 {
1023         int err;
1024         struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
1025         struct mlx4_ib_qp *mqp = to_mqp(ibqp);
1026         u64 reg_id;
1027         struct mlx4_ib_steering *ib_steering = NULL;
1028
1029         if (mdev->dev->caps.steering_mode ==
1030             MLX4_STEERING_MODE_DEVICE_MANAGED) {
1031                 ib_steering = kmalloc(sizeof(*ib_steering), GFP_KERNEL);
1032                 if (!ib_steering)
1033                         return -ENOMEM;
1034         }
1035
1036         err = mlx4_multicast_attach(mdev->dev, &mqp->mqp, gid->raw, mqp->port,
1037                                     !!(mqp->flags &
1038                                        MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK),
1039                                     MLX4_PROT_IB_IPV6, &reg_id);
1040         if (err)
1041                 goto err_malloc;
1042
1043         err = add_gid_entry(ibqp, gid);
1044         if (err)
1045                 goto err_add;
1046
1047         if (ib_steering) {
1048                 memcpy(ib_steering->gid.raw, gid->raw, 16);
1049                 ib_steering->reg_id = reg_id;
1050                 mutex_lock(&mqp->mutex);
1051                 list_add(&ib_steering->list, &mqp->steering_rules);
1052                 mutex_unlock(&mqp->mutex);
1053         }
1054         return 0;
1055
1056 err_add:
1057         mlx4_multicast_detach(mdev->dev, &mqp->mqp, gid->raw,
1058                               MLX4_PROT_IB_IPV6, reg_id);
1059 err_malloc:
1060         kfree(ib_steering);
1061
1062         return err;
1063 }
1064
1065 static struct mlx4_ib_gid_entry *find_gid_entry(struct mlx4_ib_qp *qp, u8 *raw)
1066 {
1067         struct mlx4_ib_gid_entry *ge;
1068         struct mlx4_ib_gid_entry *tmp;
1069         struct mlx4_ib_gid_entry *ret = NULL;
1070
1071         list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
1072                 if (!memcmp(raw, ge->gid.raw, 16)) {
1073                         ret = ge;
1074                         break;
1075                 }
1076         }
1077
1078         return ret;
1079 }
1080
1081 static int mlx4_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
1082 {
1083         int err;
1084         struct mlx4_ib_dev *mdev = to_mdev(ibqp->device);
1085         struct mlx4_ib_qp *mqp = to_mqp(ibqp);
1086         u8 mac[6];
1087         struct net_device *ndev;
1088         struct mlx4_ib_gid_entry *ge;
1089         u64 reg_id = 0;
1090
1091         if (mdev->dev->caps.steering_mode ==
1092             MLX4_STEERING_MODE_DEVICE_MANAGED) {
1093                 struct mlx4_ib_steering *ib_steering;
1094
1095                 mutex_lock(&mqp->mutex);
1096                 list_for_each_entry(ib_steering, &mqp->steering_rules, list) {
1097                         if (!memcmp(ib_steering->gid.raw, gid->raw, 16)) {
1098                                 list_del(&ib_steering->list);
1099                                 break;
1100                         }
1101                 }
1102                 mutex_unlock(&mqp->mutex);
1103                 if (&ib_steering->list == &mqp->steering_rules) {
1104                         pr_err("Couldn't find reg_id for mgid. Steering rule is left attached\n");
1105                         return -EINVAL;
1106                 }
1107                 reg_id = ib_steering->reg_id;
1108                 kfree(ib_steering);
1109         }
1110
1111         err = mlx4_multicast_detach(mdev->dev, &mqp->mqp, gid->raw,
1112                                     MLX4_PROT_IB_IPV6, reg_id);
1113         if (err)
1114                 return err;
1115
1116         mutex_lock(&mqp->mutex);
1117         ge = find_gid_entry(mqp, gid->raw);
1118         if (ge) {
1119                 spin_lock(&mdev->iboe.lock);
1120                 ndev = ge->added ? mdev->iboe.netdevs[ge->port - 1] : NULL;
1121                 if (ndev)
1122                         dev_hold(ndev);
1123                 spin_unlock(&mdev->iboe.lock);
1124                 rdma_get_mcast_mac((struct in6_addr *)gid, mac);
1125                 if (ndev) {
1126                         rtnl_lock();
1127                         dev_mc_del(mdev->iboe.netdevs[ge->port - 1], mac);
1128                         rtnl_unlock();
1129                         dev_put(ndev);
1130                 }
1131                 list_del(&ge->list);
1132                 kfree(ge);
1133         } else
1134                 pr_warn("could not find mgid entry\n");
1135
1136         mutex_unlock(&mqp->mutex);
1137
1138         return 0;
1139 }
1140
1141 static int init_node_data(struct mlx4_ib_dev *dev)
1142 {
1143         struct ib_smp *in_mad  = NULL;
1144         struct ib_smp *out_mad = NULL;
1145         int mad_ifc_flags = MLX4_MAD_IFC_IGNORE_KEYS;
1146         int err = -ENOMEM;
1147
1148         in_mad  = kzalloc(sizeof *in_mad, GFP_KERNEL);
1149         out_mad = kmalloc(sizeof *out_mad, GFP_KERNEL);
1150         if (!in_mad || !out_mad)
1151                 goto out;
1152
1153         init_query_mad(in_mad);
1154         in_mad->attr_id = IB_SMP_ATTR_NODE_DESC;
1155         if (mlx4_is_master(dev->dev))
1156                 mad_ifc_flags |= MLX4_MAD_IFC_NET_VIEW;
1157
1158         err = mlx4_MAD_IFC(dev, mad_ifc_flags, 1, NULL, NULL, in_mad, out_mad);
1159         if (err)
1160                 goto out;
1161
1162         memcpy(dev->ib_dev.node_desc, out_mad->data, 64);
1163
1164         in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
1165
1166         err = mlx4_MAD_IFC(dev, mad_ifc_flags, 1, NULL, NULL, in_mad, out_mad);
1167         if (err)
1168                 goto out;
1169
1170         dev->dev->rev_id = be32_to_cpup((__be32 *) (out_mad->data + 32));
1171         memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8);
1172
1173 out:
1174         kfree(in_mad);
1175         kfree(out_mad);
1176         return err;
1177 }
1178
1179 static ssize_t show_hca(struct device *device, struct device_attribute *attr,
1180                         char *buf)
1181 {
1182         struct mlx4_ib_dev *dev =
1183                 container_of(device, struct mlx4_ib_dev, ib_dev.dev);
1184         return sprintf(buf, "MT%d\n", dev->dev->pdev->device);
1185 }
1186
1187 static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr,
1188                            char *buf)
1189 {
1190         struct mlx4_ib_dev *dev =
1191                 container_of(device, struct mlx4_ib_dev, ib_dev.dev);
1192         return sprintf(buf, "%d.%d.%d\n", (int) (dev->dev->caps.fw_ver >> 32),
1193                        (int) (dev->dev->caps.fw_ver >> 16) & 0xffff,
1194                        (int) dev->dev->caps.fw_ver & 0xffff);
1195 }
1196
1197 static ssize_t show_rev(struct device *device, struct device_attribute *attr,
1198                         char *buf)
1199 {
1200         struct mlx4_ib_dev *dev =
1201                 container_of(device, struct mlx4_ib_dev, ib_dev.dev);
1202         return sprintf(buf, "%x\n", dev->dev->rev_id);
1203 }
1204
1205 static ssize_t show_board(struct device *device, struct device_attribute *attr,
1206                           char *buf)
1207 {
1208         struct mlx4_ib_dev *dev =
1209                 container_of(device, struct mlx4_ib_dev, ib_dev.dev);
1210         return sprintf(buf, "%.*s\n", MLX4_BOARD_ID_LEN,
1211                        dev->dev->board_id);
1212 }
1213
1214 static DEVICE_ATTR(hw_rev,   S_IRUGO, show_rev,    NULL);
1215 static DEVICE_ATTR(fw_ver,   S_IRUGO, show_fw_ver, NULL);
1216 static DEVICE_ATTR(hca_type, S_IRUGO, show_hca,    NULL);
1217 static DEVICE_ATTR(board_id, S_IRUGO, show_board,  NULL);
1218
1219 static struct device_attribute *mlx4_class_attributes[] = {
1220         &dev_attr_hw_rev,
1221         &dev_attr_fw_ver,
1222         &dev_attr_hca_type,
1223         &dev_attr_board_id
1224 };
1225
1226 static void mlx4_addrconf_ifid_eui48(u8 *eui, u16 vlan_id, struct net_device *dev)
1227 {
1228         memcpy(eui, dev->dev_addr, 3);
1229         memcpy(eui + 5, dev->dev_addr + 3, 3);
1230         if (vlan_id < 0x1000) {
1231                 eui[3] = vlan_id >> 8;
1232                 eui[4] = vlan_id & 0xff;
1233         } else {
1234                 eui[3] = 0xff;
1235                 eui[4] = 0xfe;
1236         }
1237         eui[0] ^= 2;
1238 }
1239
1240 static void update_gids_task(struct work_struct *work)
1241 {
1242         struct update_gid_work *gw = container_of(work, struct update_gid_work, work);
1243         struct mlx4_cmd_mailbox *mailbox;
1244         union ib_gid *gids;
1245         int err;
1246         struct mlx4_dev *dev = gw->dev->dev;
1247
1248         mailbox = mlx4_alloc_cmd_mailbox(dev);
1249         if (IS_ERR(mailbox)) {
1250                 pr_warn("update gid table failed %ld\n", PTR_ERR(mailbox));
1251                 return;
1252         }
1253
1254         gids = mailbox->buf;
1255         memcpy(gids, gw->gids, sizeof gw->gids);
1256
1257         err = mlx4_cmd(dev, mailbox->dma, MLX4_SET_PORT_GID_TABLE << 8 | gw->port,
1258                        1, MLX4_CMD_SET_PORT, MLX4_CMD_TIME_CLASS_B,
1259                        MLX4_CMD_WRAPPED);
1260         if (err)
1261                 pr_warn("set port command failed\n");
1262         else {
1263                 memcpy(gw->dev->iboe.gid_table[gw->port - 1], gw->gids, sizeof gw->gids);
1264                 mlx4_ib_dispatch_event(gw->dev, gw->port, IB_EVENT_GID_CHANGE);
1265         }
1266
1267         mlx4_free_cmd_mailbox(dev, mailbox);
1268         kfree(gw);
1269 }
1270
1271 static int update_ipv6_gids(struct mlx4_ib_dev *dev, int port, int clear)
1272 {
1273         struct net_device *ndev = dev->iboe.netdevs[port - 1];
1274         struct update_gid_work *work;
1275         struct net_device *tmp;
1276         int i;
1277         u8 *hits;
1278         int ret;
1279         union ib_gid gid;
1280         int free;
1281         int found;
1282         int need_update = 0;
1283         u16 vid;
1284
1285         work = kzalloc(sizeof *work, GFP_ATOMIC);
1286         if (!work)
1287                 return -ENOMEM;
1288
1289         hits = kzalloc(128, GFP_ATOMIC);
1290         if (!hits) {
1291                 ret = -ENOMEM;
1292                 goto out;
1293         }
1294
1295         rcu_read_lock();
1296         for_each_netdev_rcu(&init_net, tmp) {
1297                 if (ndev && (tmp == ndev || rdma_vlan_dev_real_dev(tmp) == ndev)) {
1298                         gid.global.subnet_prefix = cpu_to_be64(0xfe80000000000000LL);
1299                         vid = rdma_vlan_dev_vlan_id(tmp);
1300                         mlx4_addrconf_ifid_eui48(&gid.raw[8], vid, ndev);
1301                         found = 0;
1302                         free = -1;
1303                         for (i = 0; i < 128; ++i) {
1304                                 if (free < 0 &&
1305                                     !memcmp(&dev->iboe.gid_table[port - 1][i], &zgid, sizeof zgid))
1306                                         free = i;
1307                                 if (!memcmp(&dev->iboe.gid_table[port - 1][i], &gid, sizeof gid)) {
1308                                         hits[i] = 1;
1309                                         found = 1;
1310                                         break;
1311                                 }
1312                         }
1313
1314                         if (!found) {
1315                                 if (tmp == ndev &&
1316                                     (memcmp(&dev->iboe.gid_table[port - 1][0],
1317                                             &gid, sizeof gid) ||
1318                                      !memcmp(&dev->iboe.gid_table[port - 1][0],
1319                                              &zgid, sizeof gid))) {
1320                                         dev->iboe.gid_table[port - 1][0] = gid;
1321                                         ++need_update;
1322                                         hits[0] = 1;
1323                                 } else if (free >= 0) {
1324                                         dev->iboe.gid_table[port - 1][free] = gid;
1325                                         hits[free] = 1;
1326                                         ++need_update;
1327                                 }
1328                         }
1329                 }
1330         }
1331         rcu_read_unlock();
1332
1333         for (i = 0; i < 128; ++i)
1334                 if (!hits[i]) {
1335                         if (memcmp(&dev->iboe.gid_table[port - 1][i], &zgid, sizeof zgid))
1336                                 ++need_update;
1337                         dev->iboe.gid_table[port - 1][i] = zgid;
1338                 }
1339
1340         if (need_update) {
1341                 memcpy(work->gids, dev->iboe.gid_table[port - 1], sizeof work->gids);
1342                 INIT_WORK(&work->work, update_gids_task);
1343                 work->port = port;
1344                 work->dev = dev;
1345                 queue_work(wq, &work->work);
1346         } else
1347                 kfree(work);
1348
1349         kfree(hits);
1350         return 0;
1351
1352 out:
1353         kfree(work);
1354         return ret;
1355 }
1356
1357 static void handle_en_event(struct mlx4_ib_dev *dev, int port, unsigned long event)
1358 {
1359         switch (event) {
1360         case NETDEV_UP:
1361         case NETDEV_CHANGEADDR:
1362                 update_ipv6_gids(dev, port, 0);
1363                 break;
1364
1365         case NETDEV_DOWN:
1366                 update_ipv6_gids(dev, port, 1);
1367                 dev->iboe.netdevs[port - 1] = NULL;
1368         }
1369 }
1370
1371 static void netdev_added(struct mlx4_ib_dev *dev, int port)
1372 {
1373         update_ipv6_gids(dev, port, 0);
1374 }
1375
1376 static void netdev_removed(struct mlx4_ib_dev *dev, int port)
1377 {
1378         update_ipv6_gids(dev, port, 1);
1379 }
1380
1381 static int mlx4_ib_netdev_event(struct notifier_block *this, unsigned long event,
1382                                 void *ptr)
1383 {
1384         struct net_device *dev = netdev_notifier_info_to_dev(ptr);
1385         struct mlx4_ib_dev *ibdev;
1386         struct net_device *oldnd;
1387         struct mlx4_ib_iboe *iboe;
1388         int port;
1389
1390         if (!net_eq(dev_net(dev), &init_net))
1391                 return NOTIFY_DONE;
1392
1393         ibdev = container_of(this, struct mlx4_ib_dev, iboe.nb);
1394         iboe = &ibdev->iboe;
1395
1396         spin_lock(&iboe->lock);
1397         mlx4_foreach_ib_transport_port(port, ibdev->dev) {
1398                 oldnd = iboe->netdevs[port - 1];
1399                 iboe->netdevs[port - 1] =
1400                         mlx4_get_protocol_dev(ibdev->dev, MLX4_PROT_ETH, port);
1401                 if (oldnd != iboe->netdevs[port - 1]) {
1402                         if (iboe->netdevs[port - 1])
1403                                 netdev_added(ibdev, port);
1404                         else
1405                                 netdev_removed(ibdev, port);
1406                 }
1407         }
1408
1409         if (dev == iboe->netdevs[0] ||
1410             (iboe->netdevs[0] && rdma_vlan_dev_real_dev(dev) == iboe->netdevs[0]))
1411                 handle_en_event(ibdev, 1, event);
1412         else if (dev == iboe->netdevs[1]
1413                  || (iboe->netdevs[1] && rdma_vlan_dev_real_dev(dev) == iboe->netdevs[1]))
1414                 handle_en_event(ibdev, 2, event);
1415
1416         spin_unlock(&iboe->lock);
1417
1418         return NOTIFY_DONE;
1419 }
1420
1421 static void init_pkeys(struct mlx4_ib_dev *ibdev)
1422 {
1423         int port;
1424         int slave;
1425         int i;
1426
1427         if (mlx4_is_master(ibdev->dev)) {
1428                 for (slave = 0; slave <= ibdev->dev->num_vfs; ++slave) {
1429                         for (port = 1; port <= ibdev->dev->caps.num_ports; ++port) {
1430                                 for (i = 0;
1431                                      i < ibdev->dev->phys_caps.pkey_phys_table_len[port];
1432                                      ++i) {
1433                                         ibdev->pkeys.virt2phys_pkey[slave][port - 1][i] =
1434                                         /* master has the identity virt2phys pkey mapping */
1435                                                 (slave == mlx4_master_func_num(ibdev->dev) || !i) ? i :
1436                                                         ibdev->dev->phys_caps.pkey_phys_table_len[port] - 1;
1437                                         mlx4_sync_pkey_table(ibdev->dev, slave, port, i,
1438                                                              ibdev->pkeys.virt2phys_pkey[slave][port - 1][i]);
1439                                 }
1440                         }
1441                 }
1442                 /* initialize pkey cache */
1443                 for (port = 1; port <= ibdev->dev->caps.num_ports; ++port) {
1444                         for (i = 0;
1445                              i < ibdev->dev->phys_caps.pkey_phys_table_len[port];
1446                              ++i)
1447                                 ibdev->pkeys.phys_pkey_cache[port-1][i] =
1448                                         (i) ? 0 : 0xFFFF;
1449                 }
1450         }
1451 }
1452
1453 static void mlx4_ib_alloc_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev)
1454 {
1455         char name[32];
1456         int eq_per_port = 0;
1457         int added_eqs = 0;
1458         int total_eqs = 0;
1459         int i, j, eq;
1460
1461         /* Legacy mode or comp_pool is not large enough */
1462         if (dev->caps.comp_pool == 0 ||
1463             dev->caps.num_ports > dev->caps.comp_pool)
1464                 return;
1465
1466         eq_per_port = rounddown_pow_of_two(dev->caps.comp_pool/
1467                                         dev->caps.num_ports);
1468
1469         /* Init eq table */
1470         added_eqs = 0;
1471         mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB)
1472                 added_eqs += eq_per_port;
1473
1474         total_eqs = dev->caps.num_comp_vectors + added_eqs;
1475
1476         ibdev->eq_table = kzalloc(total_eqs * sizeof(int), GFP_KERNEL);
1477         if (!ibdev->eq_table)
1478                 return;
1479
1480         ibdev->eq_added = added_eqs;
1481
1482         eq = 0;
1483         mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_IB) {
1484                 for (j = 0; j < eq_per_port; j++) {
1485                         sprintf(name, "mlx4-ib-%d-%d@%s",
1486                                 i, j, dev->pdev->bus->name);
1487                         /* Set IRQ for specific name (per ring) */
1488                         if (mlx4_assign_eq(dev, name, NULL,
1489                                            &ibdev->eq_table[eq])) {
1490                                 /* Use legacy (same as mlx4_en driver) */
1491                                 pr_warn("Can't allocate EQ %d; reverting to legacy\n", eq);
1492                                 ibdev->eq_table[eq] =
1493                                         (eq % dev->caps.num_comp_vectors);
1494                         }
1495                         eq++;
1496                 }
1497         }
1498
1499         /* Fill the reset of the vector with legacy EQ */
1500         for (i = 0, eq = added_eqs; i < dev->caps.num_comp_vectors; i++)
1501                 ibdev->eq_table[eq++] = i;
1502
1503         /* Advertise the new number of EQs to clients */
1504         ibdev->ib_dev.num_comp_vectors = total_eqs;
1505 }
1506
1507 static void mlx4_ib_free_eqs(struct mlx4_dev *dev, struct mlx4_ib_dev *ibdev)
1508 {
1509         int i;
1510
1511         /* no additional eqs were added */
1512         if (!ibdev->eq_table)
1513                 return;
1514
1515         /* Reset the advertised EQ number */
1516         ibdev->ib_dev.num_comp_vectors = dev->caps.num_comp_vectors;
1517
1518         /* Free only the added eqs */
1519         for (i = 0; i < ibdev->eq_added; i++) {
1520                 /* Don't free legacy eqs if used */
1521                 if (ibdev->eq_table[i] <= dev->caps.num_comp_vectors)
1522                         continue;
1523                 mlx4_release_eq(dev, ibdev->eq_table[i]);
1524         }
1525
1526         kfree(ibdev->eq_table);
1527 }
1528
1529 static void *mlx4_ib_add(struct mlx4_dev *dev)
1530 {
1531         struct mlx4_ib_dev *ibdev;
1532         int num_ports = 0;
1533         int i, j;
1534         int err;
1535         struct mlx4_ib_iboe *iboe;
1536
1537         pr_info_once("%s", mlx4_ib_version);
1538
1539         mlx4_foreach_non_ib_transport_port(i, dev)
1540                 num_ports++;
1541
1542         if (mlx4_is_mfunc(dev) && num_ports) {
1543                 dev_err(&dev->pdev->dev, "RoCE is not supported over SRIOV as yet\n");
1544                 return NULL;
1545         }
1546
1547         num_ports = 0;
1548         mlx4_foreach_ib_transport_port(i, dev)
1549                 num_ports++;
1550
1551         /* No point in registering a device with no ports... */
1552         if (num_ports == 0)
1553                 return NULL;
1554
1555         ibdev = (struct mlx4_ib_dev *) ib_alloc_device(sizeof *ibdev);
1556         if (!ibdev) {
1557                 dev_err(&dev->pdev->dev, "Device struct alloc failed\n");
1558                 return NULL;
1559         }
1560
1561         iboe = &ibdev->iboe;
1562
1563         if (mlx4_pd_alloc(dev, &ibdev->priv_pdn))
1564                 goto err_dealloc;
1565
1566         if (mlx4_uar_alloc(dev, &ibdev->priv_uar))
1567                 goto err_pd;
1568
1569         ibdev->uar_map = ioremap((phys_addr_t) ibdev->priv_uar.pfn << PAGE_SHIFT,
1570                                  PAGE_SIZE);
1571         if (!ibdev->uar_map)
1572                 goto err_uar;
1573         MLX4_INIT_DOORBELL_LOCK(&ibdev->uar_lock);
1574
1575         ibdev->dev = dev;
1576
1577         strlcpy(ibdev->ib_dev.name, "mlx4_%d", IB_DEVICE_NAME_MAX);
1578         ibdev->ib_dev.owner             = THIS_MODULE;
1579         ibdev->ib_dev.node_type         = RDMA_NODE_IB_CA;
1580         ibdev->ib_dev.local_dma_lkey    = dev->caps.reserved_lkey;
1581         ibdev->num_ports                = num_ports;
1582         ibdev->ib_dev.phys_port_cnt     = ibdev->num_ports;
1583         ibdev->ib_dev.num_comp_vectors  = dev->caps.num_comp_vectors;
1584         ibdev->ib_dev.dma_device        = &dev->pdev->dev;
1585
1586         if (dev->caps.userspace_caps)
1587                 ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_ABI_VERSION;
1588         else
1589                 ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION;
1590
1591         ibdev->ib_dev.uverbs_cmd_mask   =
1592                 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT)         |
1593                 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE)        |
1594                 (1ull << IB_USER_VERBS_CMD_QUERY_PORT)          |
1595                 (1ull << IB_USER_VERBS_CMD_ALLOC_PD)            |
1596                 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD)          |
1597                 (1ull << IB_USER_VERBS_CMD_REG_MR)              |
1598                 (1ull << IB_USER_VERBS_CMD_DEREG_MR)            |
1599                 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
1600                 (1ull << IB_USER_VERBS_CMD_CREATE_CQ)           |
1601                 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ)           |
1602                 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ)          |
1603                 (1ull << IB_USER_VERBS_CMD_CREATE_QP)           |
1604                 (1ull << IB_USER_VERBS_CMD_MODIFY_QP)           |
1605                 (1ull << IB_USER_VERBS_CMD_QUERY_QP)            |
1606                 (1ull << IB_USER_VERBS_CMD_DESTROY_QP)          |
1607                 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST)        |
1608                 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST)        |
1609                 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ)          |
1610                 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ)          |
1611                 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ)           |
1612                 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ)         |
1613                 (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ)         |
1614                 (1ull << IB_USER_VERBS_CMD_OPEN_QP);
1615
1616         ibdev->ib_dev.query_device      = mlx4_ib_query_device;
1617         ibdev->ib_dev.query_port        = mlx4_ib_query_port;
1618         ibdev->ib_dev.get_link_layer    = mlx4_ib_port_link_layer;
1619         ibdev->ib_dev.query_gid         = mlx4_ib_query_gid;
1620         ibdev->ib_dev.query_pkey        = mlx4_ib_query_pkey;
1621         ibdev->ib_dev.modify_device     = mlx4_ib_modify_device;
1622         ibdev->ib_dev.modify_port       = mlx4_ib_modify_port;
1623         ibdev->ib_dev.alloc_ucontext    = mlx4_ib_alloc_ucontext;
1624         ibdev->ib_dev.dealloc_ucontext  = mlx4_ib_dealloc_ucontext;
1625         ibdev->ib_dev.mmap              = mlx4_ib_mmap;
1626         ibdev->ib_dev.alloc_pd          = mlx4_ib_alloc_pd;
1627         ibdev->ib_dev.dealloc_pd        = mlx4_ib_dealloc_pd;
1628         ibdev->ib_dev.create_ah         = mlx4_ib_create_ah;
1629         ibdev->ib_dev.query_ah          = mlx4_ib_query_ah;
1630         ibdev->ib_dev.destroy_ah        = mlx4_ib_destroy_ah;
1631         ibdev->ib_dev.create_srq        = mlx4_ib_create_srq;
1632         ibdev->ib_dev.modify_srq        = mlx4_ib_modify_srq;
1633         ibdev->ib_dev.query_srq         = mlx4_ib_query_srq;
1634         ibdev->ib_dev.destroy_srq       = mlx4_ib_destroy_srq;
1635         ibdev->ib_dev.post_srq_recv     = mlx4_ib_post_srq_recv;
1636         ibdev->ib_dev.create_qp         = mlx4_ib_create_qp;
1637         ibdev->ib_dev.modify_qp         = mlx4_ib_modify_qp;
1638         ibdev->ib_dev.query_qp          = mlx4_ib_query_qp;
1639         ibdev->ib_dev.destroy_qp        = mlx4_ib_destroy_qp;
1640         ibdev->ib_dev.post_send         = mlx4_ib_post_send;
1641         ibdev->ib_dev.post_recv         = mlx4_ib_post_recv;
1642         ibdev->ib_dev.create_cq         = mlx4_ib_create_cq;
1643         ibdev->ib_dev.modify_cq         = mlx4_ib_modify_cq;
1644         ibdev->ib_dev.resize_cq         = mlx4_ib_resize_cq;
1645         ibdev->ib_dev.destroy_cq        = mlx4_ib_destroy_cq;
1646         ibdev->ib_dev.poll_cq           = mlx4_ib_poll_cq;
1647         ibdev->ib_dev.req_notify_cq     = mlx4_ib_arm_cq;
1648         ibdev->ib_dev.get_dma_mr        = mlx4_ib_get_dma_mr;
1649         ibdev->ib_dev.reg_user_mr       = mlx4_ib_reg_user_mr;
1650         ibdev->ib_dev.dereg_mr          = mlx4_ib_dereg_mr;
1651         ibdev->ib_dev.alloc_fast_reg_mr = mlx4_ib_alloc_fast_reg_mr;
1652         ibdev->ib_dev.alloc_fast_reg_page_list = mlx4_ib_alloc_fast_reg_page_list;
1653         ibdev->ib_dev.free_fast_reg_page_list  = mlx4_ib_free_fast_reg_page_list;
1654         ibdev->ib_dev.attach_mcast      = mlx4_ib_mcg_attach;
1655         ibdev->ib_dev.detach_mcast      = mlx4_ib_mcg_detach;
1656         ibdev->ib_dev.process_mad       = mlx4_ib_process_mad;
1657
1658         if (!mlx4_is_slave(ibdev->dev)) {
1659                 ibdev->ib_dev.alloc_fmr         = mlx4_ib_fmr_alloc;
1660                 ibdev->ib_dev.map_phys_fmr      = mlx4_ib_map_phys_fmr;
1661                 ibdev->ib_dev.unmap_fmr         = mlx4_ib_unmap_fmr;
1662                 ibdev->ib_dev.dealloc_fmr       = mlx4_ib_fmr_dealloc;
1663         }
1664
1665         if (dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW ||
1666             dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN) {
1667                 ibdev->ib_dev.alloc_mw = mlx4_ib_alloc_mw;
1668                 ibdev->ib_dev.bind_mw = mlx4_ib_bind_mw;
1669                 ibdev->ib_dev.dealloc_mw = mlx4_ib_dealloc_mw;
1670
1671                 ibdev->ib_dev.uverbs_cmd_mask |=
1672                         (1ull << IB_USER_VERBS_CMD_ALLOC_MW) |
1673                         (1ull << IB_USER_VERBS_CMD_DEALLOC_MW);
1674         }
1675
1676         if (dev->caps.flags & MLX4_DEV_CAP_FLAG_XRC) {
1677                 ibdev->ib_dev.alloc_xrcd = mlx4_ib_alloc_xrcd;
1678                 ibdev->ib_dev.dealloc_xrcd = mlx4_ib_dealloc_xrcd;
1679                 ibdev->ib_dev.uverbs_cmd_mask |=
1680                         (1ull << IB_USER_VERBS_CMD_OPEN_XRCD) |
1681                         (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD);
1682         }
1683
1684         if (check_flow_steering_support(dev)) {
1685                 ibdev->ib_dev.create_flow       = mlx4_ib_create_flow;
1686                 ibdev->ib_dev.destroy_flow      = mlx4_ib_destroy_flow;
1687
1688                 ibdev->ib_dev.uverbs_ex_cmd_mask        |=
1689                         (1ull << IB_USER_VERBS_EX_CMD_CREATE_FLOW) |
1690                         (1ull << IB_USER_VERBS_EX_CMD_DESTROY_FLOW);
1691         }
1692
1693         mlx4_ib_alloc_eqs(dev, ibdev);
1694
1695         spin_lock_init(&iboe->lock);
1696
1697         if (init_node_data(ibdev))
1698                 goto err_map;
1699
1700         for (i = 0; i < ibdev->num_ports; ++i) {
1701                 if (mlx4_ib_port_link_layer(&ibdev->ib_dev, i + 1) ==
1702                                                 IB_LINK_LAYER_ETHERNET) {
1703                         err = mlx4_counter_alloc(ibdev->dev, &ibdev->counters[i]);
1704                         if (err)
1705                                 ibdev->counters[i] = -1;
1706                 } else
1707                                 ibdev->counters[i] = -1;
1708         }
1709
1710         spin_lock_init(&ibdev->sm_lock);
1711         mutex_init(&ibdev->cap_mask_mutex);
1712
1713         if (ib_register_device(&ibdev->ib_dev, NULL))
1714                 goto err_counter;
1715
1716         if (mlx4_ib_mad_init(ibdev))
1717                 goto err_reg;
1718
1719         if (mlx4_ib_init_sriov(ibdev))
1720                 goto err_mad;
1721
1722         if (dev->caps.flags & MLX4_DEV_CAP_FLAG_IBOE && !iboe->nb.notifier_call) {
1723                 iboe->nb.notifier_call = mlx4_ib_netdev_event;
1724                 err = register_netdevice_notifier(&iboe->nb);
1725                 if (err)
1726                         goto err_sriov;
1727         }
1728
1729         for (j = 0; j < ARRAY_SIZE(mlx4_class_attributes); ++j) {
1730                 if (device_create_file(&ibdev->ib_dev.dev,
1731                                        mlx4_class_attributes[j]))
1732                         goto err_notif;
1733         }
1734
1735         ibdev->ib_active = true;
1736
1737         if (mlx4_is_mfunc(ibdev->dev))
1738                 init_pkeys(ibdev);
1739
1740         /* create paravirt contexts for any VFs which are active */
1741         if (mlx4_is_master(ibdev->dev)) {
1742                 for (j = 0; j < MLX4_MFUNC_MAX; j++) {
1743                         if (j == mlx4_master_func_num(ibdev->dev))
1744                                 continue;
1745                         if (mlx4_is_slave_active(ibdev->dev, j))
1746                                 do_slave_init(ibdev, j, 1);
1747                 }
1748         }
1749         return ibdev;
1750
1751 err_notif:
1752         if (unregister_netdevice_notifier(&ibdev->iboe.nb))
1753                 pr_warn("failure unregistering notifier\n");
1754         flush_workqueue(wq);
1755
1756 err_sriov:
1757         mlx4_ib_close_sriov(ibdev);
1758
1759 err_mad:
1760         mlx4_ib_mad_cleanup(ibdev);
1761
1762 err_reg:
1763         ib_unregister_device(&ibdev->ib_dev);
1764
1765 err_counter:
1766         for (; i; --i)
1767                 if (ibdev->counters[i - 1] != -1)
1768                         mlx4_counter_free(ibdev->dev, ibdev->counters[i - 1]);
1769
1770 err_map:
1771         iounmap(ibdev->uar_map);
1772
1773 err_uar:
1774         mlx4_uar_free(dev, &ibdev->priv_uar);
1775
1776 err_pd:
1777         mlx4_pd_free(dev, ibdev->priv_pdn);
1778
1779 err_dealloc:
1780         ib_dealloc_device(&ibdev->ib_dev);
1781
1782         return NULL;
1783 }
1784
1785 static void mlx4_ib_remove(struct mlx4_dev *dev, void *ibdev_ptr)
1786 {
1787         struct mlx4_ib_dev *ibdev = ibdev_ptr;
1788         int p;
1789
1790         mlx4_ib_close_sriov(ibdev);
1791         mlx4_ib_mad_cleanup(ibdev);
1792         ib_unregister_device(&ibdev->ib_dev);
1793         if (ibdev->iboe.nb.notifier_call) {
1794                 if (unregister_netdevice_notifier(&ibdev->iboe.nb))
1795                         pr_warn("failure unregistering notifier\n");
1796                 ibdev->iboe.nb.notifier_call = NULL;
1797         }
1798         iounmap(ibdev->uar_map);
1799         for (p = 0; p < ibdev->num_ports; ++p)
1800                 if (ibdev->counters[p] != -1)
1801                         mlx4_counter_free(ibdev->dev, ibdev->counters[p]);
1802         mlx4_foreach_port(p, dev, MLX4_PORT_TYPE_IB)
1803                 mlx4_CLOSE_PORT(dev, p);
1804
1805         mlx4_ib_free_eqs(dev, ibdev);
1806
1807         mlx4_uar_free(dev, &ibdev->priv_uar);
1808         mlx4_pd_free(dev, ibdev->priv_pdn);
1809         ib_dealloc_device(&ibdev->ib_dev);
1810 }
1811
1812 static void do_slave_init(struct mlx4_ib_dev *ibdev, int slave, int do_init)
1813 {
1814         struct mlx4_ib_demux_work **dm = NULL;
1815         struct mlx4_dev *dev = ibdev->dev;
1816         int i;
1817         unsigned long flags;
1818
1819         if (!mlx4_is_master(dev))
1820                 return;
1821
1822         dm = kcalloc(dev->caps.num_ports, sizeof *dm, GFP_ATOMIC);
1823         if (!dm) {
1824                 pr_err("failed to allocate memory for tunneling qp update\n");
1825                 goto out;
1826         }
1827
1828         for (i = 0; i < dev->caps.num_ports; i++) {
1829                 dm[i] = kmalloc(sizeof (struct mlx4_ib_demux_work), GFP_ATOMIC);
1830                 if (!dm[i]) {
1831                         pr_err("failed to allocate memory for tunneling qp update work struct\n");
1832                         for (i = 0; i < dev->caps.num_ports; i++) {
1833                                 if (dm[i])
1834                                         kfree(dm[i]);
1835                         }
1836                         goto out;
1837                 }
1838         }
1839         /* initialize or tear down tunnel QPs for the slave */
1840         for (i = 0; i < dev->caps.num_ports; i++) {
1841                 INIT_WORK(&dm[i]->work, mlx4_ib_tunnels_update_work);
1842                 dm[i]->port = i + 1;
1843                 dm[i]->slave = slave;
1844                 dm[i]->do_init = do_init;
1845                 dm[i]->dev = ibdev;
1846                 spin_lock_irqsave(&ibdev->sriov.going_down_lock, flags);
1847                 if (!ibdev->sriov.is_going_down)
1848                         queue_work(ibdev->sriov.demux[i].ud_wq, &dm[i]->work);
1849                 spin_unlock_irqrestore(&ibdev->sriov.going_down_lock, flags);
1850         }
1851 out:
1852         kfree(dm);
1853         return;
1854 }
1855
1856 static void mlx4_ib_event(struct mlx4_dev *dev, void *ibdev_ptr,
1857                           enum mlx4_dev_event event, unsigned long param)
1858 {
1859         struct ib_event ibev;
1860         struct mlx4_ib_dev *ibdev = to_mdev((struct ib_device *) ibdev_ptr);
1861         struct mlx4_eqe *eqe = NULL;
1862         struct ib_event_work *ew;
1863         int p = 0;
1864
1865         if (event == MLX4_DEV_EVENT_PORT_MGMT_CHANGE)
1866                 eqe = (struct mlx4_eqe *)param;
1867         else
1868                 p = (int) param;
1869
1870         switch (event) {
1871         case MLX4_DEV_EVENT_PORT_UP:
1872                 if (p > ibdev->num_ports)
1873                         return;
1874                 if (mlx4_is_master(dev) &&
1875                     rdma_port_get_link_layer(&ibdev->ib_dev, p) ==
1876                         IB_LINK_LAYER_INFINIBAND) {
1877                         mlx4_ib_invalidate_all_guid_record(ibdev, p);
1878                 }
1879                 ibev.event = IB_EVENT_PORT_ACTIVE;
1880                 break;
1881
1882         case MLX4_DEV_EVENT_PORT_DOWN:
1883                 if (p > ibdev->num_ports)
1884                         return;
1885                 ibev.event = IB_EVENT_PORT_ERR;
1886                 break;
1887
1888         case MLX4_DEV_EVENT_CATASTROPHIC_ERROR:
1889                 ibdev->ib_active = false;
1890                 ibev.event = IB_EVENT_DEVICE_FATAL;
1891                 break;
1892
1893         case MLX4_DEV_EVENT_PORT_MGMT_CHANGE:
1894                 ew = kmalloc(sizeof *ew, GFP_ATOMIC);
1895                 if (!ew) {
1896                         pr_err("failed to allocate memory for events work\n");
1897                         break;
1898                 }
1899
1900                 INIT_WORK(&ew->work, handle_port_mgmt_change_event);
1901                 memcpy(&ew->ib_eqe, eqe, sizeof *eqe);
1902                 ew->ib_dev = ibdev;
1903                 /* need to queue only for port owner, which uses GEN_EQE */
1904                 if (mlx4_is_master(dev))
1905                         queue_work(wq, &ew->work);
1906                 else
1907                         handle_port_mgmt_change_event(&ew->work);
1908                 return;
1909
1910         case MLX4_DEV_EVENT_SLAVE_INIT:
1911                 /* here, p is the slave id */
1912                 do_slave_init(ibdev, p, 1);
1913                 return;
1914
1915         case MLX4_DEV_EVENT_SLAVE_SHUTDOWN:
1916                 /* here, p is the slave id */
1917                 do_slave_init(ibdev, p, 0);
1918                 return;
1919
1920         default:
1921                 return;
1922         }
1923
1924         ibev.device           = ibdev_ptr;
1925         ibev.element.port_num = (u8) p;
1926
1927         ib_dispatch_event(&ibev);
1928 }
1929
1930 static struct mlx4_interface mlx4_ib_interface = {
1931         .add            = mlx4_ib_add,
1932         .remove         = mlx4_ib_remove,
1933         .event          = mlx4_ib_event,
1934         .protocol       = MLX4_PROT_IB_IPV6
1935 };
1936
1937 static int __init mlx4_ib_init(void)
1938 {
1939         int err;
1940
1941         wq = create_singlethread_workqueue("mlx4_ib");
1942         if (!wq)
1943                 return -ENOMEM;
1944
1945         err = mlx4_ib_mcg_init();
1946         if (err)
1947                 goto clean_wq;
1948
1949         err = mlx4_register_interface(&mlx4_ib_interface);
1950         if (err)
1951                 goto clean_mcg;
1952
1953         return 0;
1954
1955 clean_mcg:
1956         mlx4_ib_mcg_destroy();
1957
1958 clean_wq:
1959         destroy_workqueue(wq);
1960         return err;
1961 }
1962
1963 static void __exit mlx4_ib_cleanup(void)
1964 {
1965         mlx4_unregister_interface(&mlx4_ib_interface);
1966         mlx4_ib_mcg_destroy();
1967         destroy_workqueue(wq);
1968 }
1969
1970 module_init(mlx4_ib_init);
1971 module_exit(mlx4_ib_cleanup);