drivers/infiniband/hw/amso1100/c2_qp.c

   1 /*
   2  * Copyright (c) 2004 Topspin Communications.  All rights reserved.
   3  * Copyright (c) 2005 Cisco Systems. All rights reserved.
   4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
   5  * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
   6  * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
   7  *
   8  * This software is available to you under a choice of one of two
   9  * licenses.  You may choose to be licensed under the terms of the GNU
  10  * General Public License (GPL) Version 2, available from the file
  11  * COPYING in the main directory of this source tree, or the
  12  * OpenIB.org BSD license below:
  13  *
  14  *     Redistribution and use in source and binary forms, with or
  15  *     without modification, are permitted provided that the following
  16  *     conditions are met:
  17  *
  18  *      - Redistributions of source code must retain the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer.
  21  *
  22  *      - Redistributions in binary form must reproduce the above
  23  *        copyright notice, this list of conditions and the following
  24  *        disclaimer in the documentation and/or other materials
  25  *        provided with the distribution.
  26  *
  27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  34  * SOFTWARE.
  35  *
  36  */
  37
  38 #include <linux/delay.h>
  39
  40 #include "c2.h"
  41 #include "c2_vq.h"
  42 #include "c2_status.h"
  43
  44 #define C2_MAX_ORD_PER_QP 128
  45 #define C2_MAX_IRD_PER_QP 128
  46
  47 #define C2_HINT_MAKE(q_index, hint_count) (((q_index) << 16) | hint_count)
  48 #define C2_HINT_GET_INDEX(hint) (((hint) & 0x7FFF0000) >> 16)
  49 #define C2_HINT_GET_COUNT(hint) ((hint) & 0x0000FFFF)
  50
  51 #define NO_SUPPORT -1
  52 static const u8 c2_opcode[] = {
  53         [IB_WR_SEND] = C2_WR_TYPE_SEND,
  54         [IB_WR_SEND_WITH_IMM] = NO_SUPPORT,
  55         [IB_WR_RDMA_WRITE] = C2_WR_TYPE_RDMA_WRITE,
  56         [IB_WR_RDMA_WRITE_WITH_IMM] = NO_SUPPORT,
  57         [IB_WR_RDMA_READ] = C2_WR_TYPE_RDMA_READ,
  58         [IB_WR_ATOMIC_CMP_AND_SWP] = NO_SUPPORT,
  59         [IB_WR_ATOMIC_FETCH_AND_ADD] = NO_SUPPORT,
  60 };
  61
  62 static int to_c2_state(enum ib_qp_state ib_state)
  63 {
  64         switch (ib_state) {
  65         case IB_QPS_RESET:
  66                 return C2_QP_STATE_IDLE;
  67         case IB_QPS_RTS:
  68                 return C2_QP_STATE_RTS;
  69         case IB_QPS_SQD:
  70                 return C2_QP_STATE_CLOSING;
  71         case IB_QPS_SQE:
  72                 return C2_QP_STATE_CLOSING;
  73         case IB_QPS_ERR:
  74                 return C2_QP_STATE_ERROR;
  75         default:
  76                 return -1;
  77         }
  78 }
  79
  80 static int to_ib_state(enum c2_qp_state c2_state)
  81 {
  82         switch (c2_state) {
  83         case C2_QP_STATE_IDLE:
  84                 return IB_QPS_RESET;
  85         case C2_QP_STATE_CONNECTING:
  86                 return IB_QPS_RTR;
  87         case C2_QP_STATE_RTS:
  88                 return IB_QPS_RTS;
  89         case C2_QP_STATE_CLOSING:
  90                 return IB_QPS_SQD;
  91         case C2_QP_STATE_ERROR:
  92                 return IB_QPS_ERR;
  93         case C2_QP_STATE_TERMINATE:
  94                 return IB_QPS_SQE;
  95         default:
  96                 return -1;
  97         }
  98 }
  99
 100 static const char *to_ib_state_str(int ib_state)
 101 {
 102         static const char *state_str[] = {
 103                 "IB_QPS_RESET",
 104                 "IB_QPS_INIT",
 105                 "IB_QPS_RTR",
 106                 "IB_QPS_RTS",
 107                 "IB_QPS_SQD",
 108                 "IB_QPS_SQE",
 109                 "IB_QPS_ERR"
 110         };
 111         if (ib_state < IB_QPS_RESET ||
 112             ib_state > IB_QPS_ERR)
 113                 return "<invalid IB QP state>";
 114
 115         ib_state -= IB_QPS_RESET;
 116         return state_str[ib_state];
 117 }
 118
 119 void c2_set_qp_state(struct c2_qp *qp, int c2_state)
 120 {
 121         int new_state = to_ib_state(c2_state);
 122
 123         pr_debug("%s: qp[%p] state modify %s --> %s\n",
 124                __FUNCTION__,
 125                 qp,
 126                 to_ib_state_str(qp->state),
 127                 to_ib_state_str(new_state));
 128         qp->state = new_state;
 129 }
 130
 131 #define C2_QP_NO_ATTR_CHANGE 0xFFFFFFFF
 132
 133 int c2_qp_modify(struct c2_dev *c2dev, struct c2_qp *qp,
 134                  struct ib_qp_attr *attr, int attr_mask)
 135 {
 136         struct c2wr_qp_modify_req wr;
 137         struct c2wr_qp_modify_rep *reply;
 138         struct c2_vq_req *vq_req;
 139         unsigned long flags;
 140         u8 next_state;
 141         int err;
 142
 143         pr_debug("%s:%d qp=%p, %s --> %s\n",
 144                 __FUNCTION__, __LINE__,
 145                 qp,
 146                 to_ib_state_str(qp->state),
 147                 to_ib_state_str(attr->qp_state));
 148
 149         vq_req = vq_req_alloc(c2dev);
 150         if (!vq_req)
 151                 return -ENOMEM;
 152
 153         c2_wr_set_id(&wr, CCWR_QP_MODIFY);
 154         wr.hdr.context = (unsigned long) vq_req;
 155         wr.rnic_handle = c2dev->adapter_handle;
 156         wr.qp_handle = qp->adapter_handle;
 157         wr.ord = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 158         wr.ird = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 159         wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 160         wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 161
 162         if (attr_mask & IB_QP_STATE) {
 163                 /* Ensure the state is valid */
 164                 if (attr->qp_state < 0 || attr->qp_state > IB_QPS_ERR)
 165                         return -EINVAL;
 166
 167                 wr.next_qp_state = cpu_to_be32(to_c2_state(attr->qp_state));
 168
 169                 if (attr->qp_state == IB_QPS_ERR) {
 170                         spin_lock_irqsave(&qp->lock, flags);
 171                         if (qp->cm_id && qp->state == IB_QPS_RTS) {
 172                                 pr_debug("Generating CLOSE event for QP-->ERR, "
 173                                         "qp=%p, cm_id=%p\n",qp,qp->cm_id);
 174                                 /* Generate an CLOSE event */
 175                                 vq_req->cm_id = qp->cm_id;
 176                                 vq_req->event = IW_CM_EVENT_CLOSE;
 177                         }
 178                         spin_unlock_irqrestore(&qp->lock, flags);
 179                 }
 180                 next_state =  attr->qp_state;
 181
 182         } else if (attr_mask & IB_QP_CUR_STATE) {
 183
 184                 if (attr->cur_qp_state != IB_QPS_RTR &&
 185                     attr->cur_qp_state != IB_QPS_RTS &&
 186                     attr->cur_qp_state != IB_QPS_SQD &&
 187                     attr->cur_qp_state != IB_QPS_SQE)
 188                         return -EINVAL;
 189                 else
 190                         wr.next_qp_state =
 191                             cpu_to_be32(to_c2_state(attr->cur_qp_state));
 192
 193                 next_state = attr->cur_qp_state;
 194
 195         } else {
 196                 err = 0;
 197                 goto bail0;
 198         }
 199
 200         /* reference the request struct */
 201         vq_req_get(c2dev, vq_req);
 202
 203         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 204         if (err) {
 205                 vq_req_put(c2dev, vq_req);
 206                 goto bail0;
 207         }
 208
 209         err = vq_wait_for_reply(c2dev, vq_req);
 210         if (err)
 211                 goto bail0;
 212
 213         reply = (struct c2wr_qp_modify_rep *) (unsigned long) vq_req->reply_msg;
 214         if (!reply) {
 215                 err = -ENOMEM;
 216                 goto bail0;
 217         }
 218
 219         err = c2_errno(reply);
 220         if (!err)
 221                 qp->state = next_state;
 222 #ifdef DEBUG
 223         else
 224                 pr_debug("%s: c2_errno=%d\n", __FUNCTION__, err);
 225 #endif
 226         /*
 227          * If we're going to error and generating the event here, then
 228          * we need to remove the reference because there will be no
 229          * close event generated by the adapter
 230         */
 231         spin_lock_irqsave(&qp->lock, flags);
 232         if (vq_req->event==IW_CM_EVENT_CLOSE && qp->cm_id) {
 233                 qp->cm_id->rem_ref(qp->cm_id);
 234                 qp->cm_id = NULL;
 235         }
 236         spin_unlock_irqrestore(&qp->lock, flags);
 237
 238         vq_repbuf_free(c2dev, reply);
 239       bail0:
 240         vq_req_free(c2dev, vq_req);
 241
 242         pr_debug("%s:%d qp=%p, cur_state=%s\n",
 243                 __FUNCTION__, __LINE__,
 244                 qp,
 245                 to_ib_state_str(qp->state));
 246         return err;
 247 }
 248
 249 int c2_qp_set_read_limits(struct c2_dev *c2dev, struct c2_qp *qp,
 250                           int ord, int ird)
 251 {
 252         struct c2wr_qp_modify_req wr;
 253         struct c2wr_qp_modify_rep *reply;
 254         struct c2_vq_req *vq_req;
 255         int err;
 256
 257         vq_req = vq_req_alloc(c2dev);
 258         if (!vq_req)
 259                 return -ENOMEM;
 260
 261         c2_wr_set_id(&wr, CCWR_QP_MODIFY);
 262         wr.hdr.context = (unsigned long) vq_req;
 263         wr.rnic_handle = c2dev->adapter_handle;
 264         wr.qp_handle = qp->adapter_handle;
 265         wr.ord = cpu_to_be32(ord);
 266         wr.ird = cpu_to_be32(ird);
 267         wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 268         wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 269         wr.next_qp_state = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
 270
 271         /* reference the request struct */
 272         vq_req_get(c2dev, vq_req);
 273
 274         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 275         if (err) {
 276                 vq_req_put(c2dev, vq_req);
 277                 goto bail0;
 278         }
 279
 280         err = vq_wait_for_reply(c2dev, vq_req);
 281         if (err)
 282                 goto bail0;
 283
 284         reply = (struct c2wr_qp_modify_rep *) (unsigned long)
 285                 vq_req->reply_msg;
 286         if (!reply) {
 287                 err = -ENOMEM;
 288                 goto bail0;
 289         }
 290
 291         err = c2_errno(reply);
 292         vq_repbuf_free(c2dev, reply);
 293       bail0:
 294         vq_req_free(c2dev, vq_req);
 295         return err;
 296 }
 297
 298 static int destroy_qp(struct c2_dev *c2dev, struct c2_qp *qp)
 299 {
 300         struct c2_vq_req *vq_req;
 301         struct c2wr_qp_destroy_req wr;
 302         struct c2wr_qp_destroy_rep *reply;
 303         unsigned long flags;
 304         int err;
 305
 306         /*
 307          * Allocate a verb request message
 308          */
 309         vq_req = vq_req_alloc(c2dev);
 310         if (!vq_req) {
 311                 return -ENOMEM;
 312         }
 313
 314         /*
 315          * Initialize the WR
 316          */
 317         c2_wr_set_id(&wr, CCWR_QP_DESTROY);
 318         wr.hdr.context = (unsigned long) vq_req;
 319         wr.rnic_handle = c2dev->adapter_handle;
 320         wr.qp_handle = qp->adapter_handle;
 321
 322         /*
 323          * reference the request struct.  dereferenced in the int handler.
 324          */
 325         vq_req_get(c2dev, vq_req);
 326
 327         spin_lock_irqsave(&qp->lock, flags);
 328         if (qp->cm_id && qp->state == IB_QPS_RTS) {
 329                 pr_debug("destroy_qp: generating CLOSE event for QP-->ERR, "
 330                         "qp=%p, cm_id=%p\n",qp,qp->cm_id);
 331                 /* Generate an CLOSE event */
 332                 vq_req->qp = qp;
 333                 vq_req->cm_id = qp->cm_id;
 334                 vq_req->event = IW_CM_EVENT_CLOSE;
 335         }
 336         spin_unlock_irqrestore(&qp->lock, flags);
 337
 338         /*
 339          * Send WR to adapter
 340          */
 341         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 342         if (err) {
 343                 vq_req_put(c2dev, vq_req);
 344                 goto bail0;
 345         }
 346
 347         /*
 348          * Wait for reply from adapter
 349          */
 350         err = vq_wait_for_reply(c2dev, vq_req);
 351         if (err) {
 352                 goto bail0;
 353         }
 354
 355         /*
 356          * Process reply
 357          */
 358         reply = (struct c2wr_qp_destroy_rep *) (unsigned long) (vq_req->reply_msg);
 359         if (!reply) {
 360                 err = -ENOMEM;
 361                 goto bail0;
 362         }
 363
 364         spin_lock_irqsave(&qp->lock, flags);
 365         if (qp->cm_id) {
 366                 qp->cm_id->rem_ref(qp->cm_id);
 367                 qp->cm_id = NULL;
 368         }
 369         spin_unlock_irqrestore(&qp->lock, flags);
 370
 371         vq_repbuf_free(c2dev, reply);
 372       bail0:
 373         vq_req_free(c2dev, vq_req);
 374         return err;
 375 }
 376
 377 static int c2_alloc_qpn(struct c2_dev *c2dev, struct c2_qp *qp)
 378 {
 379         int ret;
 380
 381         do {
 382                 spin_lock_irq(&c2dev->qp_table.lock);
 383                 ret = idr_get_new_above(&c2dev->qp_table.idr, qp,
 384                                         c2dev->qp_table.last++, &qp->qpn);
 385                 spin_unlock_irq(&c2dev->qp_table.lock);
 386         } while ((ret == -EAGAIN) &&
 387                  idr_pre_get(&c2dev->qp_table.idr, GFP_KERNEL));
 388         return ret;
 389 }
 390
 391 static void c2_free_qpn(struct c2_dev *c2dev, int qpn)
 392 {
 393         spin_lock_irq(&c2dev->qp_table.lock);
 394         idr_remove(&c2dev->qp_table.idr, qpn);
 395         spin_unlock_irq(&c2dev->qp_table.lock);
 396 }
 397
 398 struct c2_qp *c2_find_qpn(struct c2_dev *c2dev, int qpn)
 399 {
 400         unsigned long flags;
 401         struct c2_qp *qp;
 402
 403         spin_lock_irqsave(&c2dev->qp_table.lock, flags);
 404         qp = idr_find(&c2dev->qp_table.idr, qpn);
 405         spin_unlock_irqrestore(&c2dev->qp_table.lock, flags);
 406         return qp;
 407 }
 408
 409 int c2_alloc_qp(struct c2_dev *c2dev,
 410                 struct c2_pd *pd,
 411                 struct ib_qp_init_attr *qp_attrs, struct c2_qp *qp)
 412 {
 413         struct c2wr_qp_create_req wr;
 414         struct c2wr_qp_create_rep *reply;
 415         struct c2_vq_req *vq_req;
 416         struct c2_cq *send_cq = to_c2cq(qp_attrs->send_cq);
 417         struct c2_cq *recv_cq = to_c2cq(qp_attrs->recv_cq);
 418         unsigned long peer_pa;
 419         u32 q_size, msg_size, mmap_size;
 420         void __iomem *mmap;
 421         int err;
 422
 423         err = c2_alloc_qpn(c2dev, qp);
 424         if (err)
 425                 return err;
 426         qp->ibqp.qp_num = qp->qpn;
 427         qp->ibqp.qp_type = IB_QPT_RC;
 428
 429         /* Allocate the SQ and RQ shared pointers */
 430         qp->sq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
 431                                          &qp->sq_mq.shared_dma, GFP_KERNEL);
 432         if (!qp->sq_mq.shared) {
 433                 err = -ENOMEM;
 434                 goto bail0;
 435         }
 436
 437         qp->rq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
 438                                          &qp->rq_mq.shared_dma, GFP_KERNEL);
 439         if (!qp->rq_mq.shared) {
 440                 err = -ENOMEM;
 441                 goto bail1;
 442         }
 443
 444         /* Allocate the verbs request */
 445         vq_req = vq_req_alloc(c2dev);
 446         if (vq_req == NULL) {
 447                 err = -ENOMEM;
 448                 goto bail2;
 449         }
 450
 451         /* Initialize the work request */
 452         memset(&wr, 0, sizeof(wr));
 453         c2_wr_set_id(&wr, CCWR_QP_CREATE);
 454         wr.hdr.context = (unsigned long) vq_req;
 455         wr.rnic_handle = c2dev->adapter_handle;
 456         wr.sq_cq_handle = send_cq->adapter_handle;
 457         wr.rq_cq_handle = recv_cq->adapter_handle;
 458         wr.sq_depth = cpu_to_be32(qp_attrs->cap.max_send_wr + 1);
 459         wr.rq_depth = cpu_to_be32(qp_attrs->cap.max_recv_wr + 1);
 460         wr.srq_handle = 0;
 461         wr.flags = cpu_to_be32(QP_RDMA_READ | QP_RDMA_WRITE | QP_MW_BIND |
 462                                QP_ZERO_STAG | QP_RDMA_READ_RESPONSE);
 463         wr.send_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
 464         wr.recv_sgl_depth = cpu_to_be32(qp_attrs->cap.max_recv_sge);
 465         wr.rdma_write_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
 466         wr.shared_sq_ht = cpu_to_be64(qp->sq_mq.shared_dma);
 467         wr.shared_rq_ht = cpu_to_be64(qp->rq_mq.shared_dma);
 468         wr.ord = cpu_to_be32(C2_MAX_ORD_PER_QP);
 469         wr.ird = cpu_to_be32(C2_MAX_IRD_PER_QP);
 470         wr.pd_id = pd->pd_id;
 471         wr.user_context = (unsigned long) qp;
 472
 473         vq_req_get(c2dev, vq_req);
 474
 475         /* Send the WR to the adapter */
 476         err = vq_send_wr(c2dev, (union c2wr *) & wr);
 477         if (err) {
 478                 vq_req_put(c2dev, vq_req);
 479                 goto bail3;
 480         }
 481
 482         /* Wait for the verb reply  */
 483         err = vq_wait_for_reply(c2dev, vq_req);
 484         if (err) {
 485                 goto bail3;
 486         }
 487
 488         /* Process the reply */
 489         reply = (struct c2wr_qp_create_rep *) (unsigned long) (vq_req->reply_msg);
 490         if (!reply) {
 491                 err = -ENOMEM;
 492                 goto bail3;
 493         }
 494
 495         if ((err = c2_wr_get_result(reply)) != 0) {
 496                 goto bail4;
 497         }
 498
 499         /* Fill in the kernel QP struct */
 500         atomic_set(&qp->refcount, 1);
 501         qp->adapter_handle = reply->qp_handle;
 502         qp->state = IB_QPS_RESET;
 503         qp->send_sgl_depth = qp_attrs->cap.max_send_sge;
 504         qp->rdma_write_sgl_depth = qp_attrs->cap.max_send_sge;
 505         qp->recv_sgl_depth = qp_attrs->cap.max_recv_sge;
 506
 507         /* Initialize the SQ MQ */
 508         q_size = be32_to_cpu(reply->sq_depth);
 509         msg_size = be32_to_cpu(reply->sq_msg_size);
 510         peer_pa = c2dev->pa + be32_to_cpu(reply->sq_mq_start);
 511         mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
 512         mmap = ioremap_nocache(peer_pa, mmap_size);
 513         if (!mmap) {
 514                 err = -ENOMEM;
 515                 goto bail5;
 516         }
 517
 518         c2_mq_req_init(&qp->sq_mq,
 519                        be32_to_cpu(reply->sq_mq_index),
 520                        q_size,
 521                        msg_size,
 522                        mmap + sizeof(struct c2_mq_shared),      /* pool start */
 523                        mmap,                            /* peer */
 524                        C2_MQ_ADAPTER_TARGET);
 525
 526         /* Initialize the RQ mq */
 527         q_size = be32_to_cpu(reply->rq_depth);
 528         msg_size = be32_to_cpu(reply->rq_msg_size);
 529         peer_pa = c2dev->pa + be32_to_cpu(reply->rq_mq_start);
 530         mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
 531         mmap = ioremap_nocache(peer_pa, mmap_size);
 532         if (!mmap) {
 533                 err = -ENOMEM;
 534                 goto bail6;
 535         }
 536
 537         c2_mq_req_init(&qp->rq_mq,
 538                        be32_to_cpu(reply->rq_mq_index),
 539                        q_size,
 540                        msg_size,
 541                        mmap + sizeof(struct c2_mq_shared),      /* pool start */
 542                        mmap,                            /* peer */
 543                        C2_MQ_ADAPTER_TARGET);
 544
 545         vq_repbuf_free(c2dev, reply);
 546         vq_req_free(c2dev, vq_req);
 547
 548         return 0;
 549
 550       bail6:
 551         iounmap(qp->sq_mq.peer);
 552       bail5:
 553         destroy_qp(c2dev, qp);
 554       bail4:
 555         vq_repbuf_free(c2dev, reply);
 556       bail3:
 557         vq_req_free(c2dev, vq_req);
 558       bail2:
 559         c2_free_mqsp(qp->rq_mq.shared);
 560       bail1:
 561         c2_free_mqsp(qp->sq_mq.shared);
 562       bail0:
 563         c2_free_qpn(c2dev, qp->qpn);
 564         return err;
 565 }
 566
 567 void c2_free_qp(struct c2_dev *c2dev, struct c2_qp *qp)
 568 {
 569         struct c2_cq *send_cq;
 570         struct c2_cq *recv_cq;
 571
 572         send_cq = to_c2cq(qp->ibqp.send_cq);
 573         recv_cq = to_c2cq(qp->ibqp.recv_cq);
 574
 575         /*
 576          * Lock CQs here, so that CQ polling code can do QP lookup
 577          * without taking a lock.
 578          */
 579         spin_lock_irq(&send_cq->lock);
 580         if (send_cq != recv_cq)
 581                 spin_lock(&recv_cq->lock);
 582
 583         c2_free_qpn(c2dev, qp->qpn);
 584
 585         if (send_cq != recv_cq)
 586                 spin_unlock(&recv_cq->lock);
 587         spin_unlock_irq(&send_cq->lock);
 588
 589         /*
 590          * Destory qp in the rnic...
 591          */
 592         destroy_qp(c2dev, qp);
 593
 594         /*
 595          * Mark any unreaped CQEs as null and void.
 596          */
 597         c2_cq_clean(c2dev, qp, send_cq->cqn);
 598         if (send_cq != recv_cq)
 599                 c2_cq_clean(c2dev, qp, recv_cq->cqn);
 600         /*
 601          * Unmap the MQs and return the shared pointers
 602          * to the message pool.
 603          */
 604         iounmap(qp->sq_mq.peer);
 605         iounmap(qp->rq_mq.peer);
 606         c2_free_mqsp(qp->sq_mq.shared);
 607         c2_free_mqsp(qp->rq_mq.shared);
 608
 609         atomic_dec(&qp->refcount);
 610         wait_event(qp->wait, !atomic_read(&qp->refcount));
 611 }
 612
 613 /*
 614  * Function: move_sgl
 615  *
 616  * Description:
 617  * Move an SGL from the user's work request struct into a CCIL Work Request
 618  * message, swapping to WR byte order and ensure the total length doesn't
 619  * overflow.
 620  *
 621  * IN:
 622  * dst          - ptr to CCIL Work Request message SGL memory.
 623  * src          - ptr to the consumers SGL memory.
 624  *
 625  * OUT: none
 626  *
 627  * Return:
 628  * CCIL status codes.
 629  */
 630 static int
 631 move_sgl(struct c2_data_addr * dst, struct ib_sge *src, int count, u32 * p_len,
 632          u8 * actual_count)
 633 {
 634         u32 tot = 0;            /* running total */
 635         u8 acount = 0;          /* running total non-0 len sge's */
 636
 637         while (count > 0) {
 638                 /*
 639                  * If the addition of this SGE causes the
 640                  * total SGL length to exceed 2^32-1, then
 641                  * fail-n-bail.
 642                  *
 643                  * If the current total plus the next element length
 644                  * wraps, then it will go negative and be less than the
 645                  * current total...
 646                  */
 647                 if ((tot + src->length) < tot) {
 648                         return -EINVAL;
 649                 }
 650                 /*
 651                  * Bug: 1456 (as well as 1498 & 1643)
 652                  * Skip over any sge's supplied with len=0
 653                  */
 654                 if (src->length) {
 655                         tot += src->length;
 656                         dst->stag = cpu_to_be32(src->lkey);
 657                         dst->to = cpu_to_be64(src->addr);
 658                         dst->length = cpu_to_be32(src->length);
 659                         dst++;
 660                         acount++;
 661                 }
 662                 src++;
 663                 count--;
 664         }
 665
 666         if (acount == 0) {
 667                 /*
 668                  * Bug: 1476 (as well as 1498, 1456 and 1643)
 669                  * Setup the SGL in the WR to make it easier for the RNIC.
 670                  * This way, the FW doesn't have to deal with special cases.
 671                  * Setting length=0 should be sufficient.
 672                  */
 673                 dst->stag = 0;
 674                 dst->to = 0;
 675                 dst->length = 0;
 676         }
 677
 678         *p_len = tot;
 679         *actual_count = acount;
 680         return 0;
 681 }
 682
 683 /*
 684  * Function: c2_activity (private function)
 685  *
 686  * Description:
 687  * Post an mq index to the host->adapter activity fifo.
 688  *
 689  * IN:
 690  * c2dev        - ptr to c2dev structure
 691  * mq_index     - mq index to post
 692  * shared       - value most recently written to shared
 693  *
 694  * OUT:
 695  *
 696  * Return:
 697  * none
 698  */
 699 static inline void c2_activity(struct c2_dev *c2dev, u32 mq_index, u16 shared)
 700 {
 701         /*
 702          * First read the register to see if the FIFO is full, and if so,
 703          * spin until it's not.  This isn't perfect -- there is no
 704          * synchronization among the clients of the register, but in
 705          * practice it prevents multiple CPU from hammering the bus
 706          * with PCI RETRY. Note that when this does happen, the card
 707          * cannot get on the bus and the card and system hang in a
 708          * deadlock -- thus the need for this code. [TOT]
 709          */
 710         while (readl(c2dev->regs + PCI_BAR0_ADAPTER_HINT) & 0x80000000)
 711                 udelay(10);
 712
 713         __raw_writel(C2_HINT_MAKE(mq_index, shared),
 714                      c2dev->regs + PCI_BAR0_ADAPTER_HINT);
 715 }
 716
 717 /*
 718  * Function: qp_wr_post
 719  *
 720  * Description:
 721  * This in-line function allocates a MQ msg, then moves the host-copy of
 722  * the completed WR into msg.  Then it posts the message.
 723  *
 724  * IN:
 725  * q            - ptr to user MQ.
 726  * wr           - ptr to host-copy of the WR.
 727  * qp           - ptr to user qp
 728  * size         - Number of bytes to post.  Assumed to be divisible by 4.
 729  *
 730  * OUT: none
 731  *
 732  * Return:
 733  * CCIL status codes.
 734  */
 735 static int qp_wr_post(struct c2_mq *q, union c2wr * wr, struct c2_qp *qp, u32 size)
 736 {
 737         union c2wr *msg;
 738
 739         msg = c2_mq_alloc(q);
 740         if (msg == NULL) {
 741                 return -EINVAL;
 742         }
 743 #ifdef CCMSGMAGIC
 744         ((c2wr_hdr_t *) wr)->magic = cpu_to_be32(CCWR_MAGIC);
 745 #endif
 746
 747         /*
 748          * Since all header fields in the WR are the same as the
 749          * CQE, set the following so the adapter need not.
 750          */
 751         c2_wr_set_result(wr, CCERR_PENDING);
 752
 753         /*
 754          * Copy the wr down to the adapter
 755          */
 756         memcpy((void *) msg, (void *) wr, size);
 757
 758         c2_mq_produce(q);
 759         return 0;
 760 }
 761
 762
 763 int c2_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
 764                  struct ib_send_wr **bad_wr)
 765 {
 766         struct c2_dev *c2dev = to_c2dev(ibqp->device);
 767         struct c2_qp *qp = to_c2qp(ibqp);
 768         union c2wr wr;
 769         unsigned long lock_flags;
 770         int err = 0;
 771
 772         u32 flags;
 773         u32 tot_len;
 774         u8 actual_sge_count;
 775         u32 msg_size;
 776
 777         if (qp->state > IB_QPS_RTS)
 778                 return -EINVAL;
 779
 780         while (ib_wr) {
 781
 782                 flags = 0;
 783                 wr.sqwr.sq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
 784                 if (ib_wr->send_flags & IB_SEND_SIGNALED) {
 785                         flags |= SQ_SIGNALED;
 786                 }
 787
 788                 switch (ib_wr->opcode) {
 789                 case IB_WR_SEND:
 790                         if (ib_wr->send_flags & IB_SEND_SOLICITED) {
 791                                 c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE);
 792                                 msg_size = sizeof(struct c2wr_send_req);
 793                         } else {
 794                                 c2_wr_set_id(&wr, C2_WR_TYPE_SEND);
 795                                 msg_size = sizeof(struct c2wr_send_req);
 796                         }
 797
 798                         wr.sqwr.send.remote_stag = 0;
 799                         msg_size += sizeof(struct c2_data_addr) * ib_wr->num_sge;
 800                         if (ib_wr->num_sge > qp->send_sgl_depth) {
 801                                 err = -EINVAL;
 802                                 break;
 803                         }
 804                         if (ib_wr->send_flags & IB_SEND_FENCE) {
 805                                 flags |= SQ_READ_FENCE;
 806                         }
 807                         err = move_sgl((struct c2_data_addr *) & (wr.sqwr.send.data),
 808                                        ib_wr->sg_list,
 809                                        ib_wr->num_sge,
 810                                        &tot_len, &actual_sge_count);
 811                         wr.sqwr.send.sge_len = cpu_to_be32(tot_len);
 812                         c2_wr_set_sge_count(&wr, actual_sge_count);
 813                         break;
 814                 case IB_WR_RDMA_WRITE:
 815                         c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_WRITE);
 816                         msg_size = sizeof(struct c2wr_rdma_write_req) +
 817                             (sizeof(struct c2_data_addr) * ib_wr->num_sge);
 818                         if (ib_wr->num_sge > qp->rdma_write_sgl_depth) {
 819                                 err = -EINVAL;
 820                                 break;
 821                         }
 822                         if (ib_wr->send_flags & IB_SEND_FENCE) {
 823                                 flags |= SQ_READ_FENCE;
 824                         }
 825                         wr.sqwr.rdma_write.remote_stag =
 826                             cpu_to_be32(ib_wr->wr.rdma.rkey);
 827                         wr.sqwr.rdma_write.remote_to =
 828                             cpu_to_be64(ib_wr->wr.rdma.remote_addr);
 829                         err = move_sgl((struct c2_data_addr *)
 830                                        & (wr.sqwr.rdma_write.data),
 831                                        ib_wr->sg_list,
 832                                        ib_wr->num_sge,
 833                                        &tot_len, &actual_sge_count);
 834                         wr.sqwr.rdma_write.sge_len = cpu_to_be32(tot_len);
 835                         c2_wr_set_sge_count(&wr, actual_sge_count);
 836                         break;
 837                 case IB_WR_RDMA_READ:
 838                         c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_READ);
 839                         msg_size = sizeof(struct c2wr_rdma_read_req);
 840
 841                         /* IWarp only suppots 1 sge for RDMA reads */
 842                         if (ib_wr->num_sge > 1) {
 843                                 err = -EINVAL;
 844                                 break;
 845                         }
 846
 847                         /*
 848                          * Move the local and remote stag/to/len into the WR.
 849                          */
 850                         wr.sqwr.rdma_read.local_stag =
 851                             cpu_to_be32(ib_wr->sg_list->lkey);
 852                         wr.sqwr.rdma_read.local_to =
 853                             cpu_to_be64(ib_wr->sg_list->addr);
 854                         wr.sqwr.rdma_read.remote_stag =
 855                             cpu_to_be32(ib_wr->wr.rdma.rkey);
 856                         wr.sqwr.rdma_read.remote_to =
 857                             cpu_to_be64(ib_wr->wr.rdma.remote_addr);
 858                         wr.sqwr.rdma_read.length =
 859                             cpu_to_be32(ib_wr->sg_list->length);
 860                         break;
 861                 default:
 862                         /* error */
 863                         msg_size = 0;
 864                         err = -EINVAL;
 865                         break;
 866                 }
 867
 868                 /*
 869                  * If we had an error on the last wr build, then
 870                  * break out.  Possible errors include bogus WR
 871                  * type, and a bogus SGL length...
 872                  */
 873                 if (err) {
 874                         break;
 875                 }
 876
 877                 /*
 878                  * Store flags
 879                  */
 880                 c2_wr_set_flags(&wr, flags);
 881
 882                 /*
 883                  * Post the puppy!
 884                  */
 885                 spin_lock_irqsave(&qp->lock, lock_flags);
 886                 err = qp_wr_post(&qp->sq_mq, &wr, qp, msg_size);
 887                 if (err) {
 888                         spin_unlock_irqrestore(&qp->lock, lock_flags);
 889                         break;
 890                 }
 891
 892                 /*
 893                  * Enqueue mq index to activity FIFO.
 894                  */
 895                 c2_activity(c2dev, qp->sq_mq.index, qp->sq_mq.hint_count);
 896                 spin_unlock_irqrestore(&qp->lock, lock_flags);
 897
 898                 ib_wr = ib_wr->next;
 899         }
 900
 901         if (err)
 902                 *bad_wr = ib_wr;
 903         return err;
 904 }
 905
 906 int c2_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr,
 907                     struct ib_recv_wr **bad_wr)
 908 {
 909         struct c2_dev *c2dev = to_c2dev(ibqp->device);
 910         struct c2_qp *qp = to_c2qp(ibqp);
 911         union c2wr wr;
 912         unsigned long lock_flags;
 913         int err = 0;
 914
 915         if (qp->state > IB_QPS_RTS)
 916                 return -EINVAL;
 917
 918         /*
 919          * Try and post each work request
 920          */
 921         while (ib_wr) {
 922                 u32 tot_len;
 923                 u8 actual_sge_count;
 924
 925                 if (ib_wr->num_sge > qp->recv_sgl_depth) {
 926                         err = -EINVAL;
 927                         break;
 928                 }
 929
 930                 /*
 931                  * Create local host-copy of the WR
 932                  */
 933                 wr.rqwr.rq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
 934                 c2_wr_set_id(&wr, CCWR_RECV);
 935                 c2_wr_set_flags(&wr, 0);
 936
 937                 /* sge_count is limited to eight bits. */
 938                 BUG_ON(ib_wr->num_sge >= 256);
 939                 err = move_sgl((struct c2_data_addr *) & (wr.rqwr.data),
 940                                ib_wr->sg_list,
 941                                ib_wr->num_sge, &tot_len, &actual_sge_count);
 942                 c2_wr_set_sge_count(&wr, actual_sge_count);
 943
 944                 /*
 945                  * If we had an error on the last wr build, then
 946                  * break out.  Possible errors include bogus WR
 947                  * type, and a bogus SGL length...
 948                  */
 949                 if (err) {
 950                         break;
 951                 }
 952
 953                 spin_lock_irqsave(&qp->lock, lock_flags);
 954                 err = qp_wr_post(&qp->rq_mq, &wr, qp, qp->rq_mq.msg_size);
 955                 if (err) {
 956                         spin_unlock_irqrestore(&qp->lock, lock_flags);
 957                         break;
 958                 }
 959
 960                 /*
 961                  * Enqueue mq index to activity FIFO
 962                  */
 963                 c2_activity(c2dev, qp->rq_mq.index, qp->rq_mq.hint_count);
 964                 spin_unlock_irqrestore(&qp->lock, lock_flags);
 965
 966                 ib_wr = ib_wr->next;
 967         }
 968
 969         if (err)
 970                 *bad_wr = ib_wr;
 971         return err;
 972 }
 973
 974 void __devinit c2_init_qp_table(struct c2_dev *c2dev)
 975 {
 976         spin_lock_init(&c2dev->qp_table.lock);
 977         idr_init(&c2dev->qp_table.idr);
 978 }
 979
 980 void __devexit c2_cleanup_qp_table(struct c2_dev *c2dev)
 981 {
 982         idr_destroy(&c2dev->qp_table.idr);
 983 }