drivers/scsi/iscsi_tcp.c

   1 /*
   2  * iSCSI Initiator over TCP/IP Data-Path
   3  *
   4  * Copyright (C) 2004 Dmitry Yusupov
   5  * Copyright (C) 2004 Alex Aizman
   6  * Copyright (C) 2005 Mike Christie
   7  * maintained by open-iscsi@googlegroups.com
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published
  11  * by the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful, but
  15  * WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  17  * General Public License for more details.
  18  *
  19  * See the file COPYING included with this distribution for more details.
  20  *
  21  * Credits:
  22  *      Christoph Hellwig
  23  *      FUJITA Tomonori
  24  *      Arne Redlich
  25  *      Zhenyu Wang
  26  */
  27
  28 #include <linux/types.h>
  29 #include <linux/list.h>
  30 #include <linux/inet.h>
  31 #include <linux/blkdev.h>
  32 #include <linux/crypto.h>
  33 #include <linux/delay.h>
  34 #include <linux/kfifo.h>
  35 #include <linux/scatterlist.h>
  36 #include <linux/mutex.h>
  37 #include <net/tcp.h>
  38 #include <scsi/scsi_cmnd.h>
  39 #include <scsi/scsi_device.h>
  40 #include <scsi/scsi_eh.h>
  41 #include <scsi/scsi_request.h>
  42 #include <scsi/scsi_tcq.h>
  43 #include <scsi/scsi_host.h>
  44 #include <scsi/scsi.h>
  45 #include <scsi/scsi_transport_iscsi.h>
  46
  47 #include "iscsi_tcp.h"
  48
  49 MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, "
  50               "Alex Aizman <itn780@yahoo.com>");
  51 MODULE_DESCRIPTION("iSCSI/TCP data-path");
  52 MODULE_LICENSE("GPL");
  53 MODULE_VERSION("0:4.445");
  54 /* #define DEBUG_TCP */
  55 /* #define DEBUG_SCSI */
  56 #define DEBUG_ASSERT
  57
  58 #ifdef DEBUG_TCP
  59 #define debug_tcp(fmt...) printk(KERN_DEBUG "tcp: " fmt)
  60 #else
  61 #define debug_tcp(fmt...)
  62 #endif
  63
  64 #ifdef DEBUG_SCSI
  65 #define debug_scsi(fmt...) printk(KERN_DEBUG "scsi: " fmt)
  66 #else
  67 #define debug_scsi(fmt...)
  68 #endif
  69
  70 #ifndef DEBUG_ASSERT
  71 #ifdef BUG_ON
  72 #undef BUG_ON
  73 #endif
  74 #define BUG_ON(expr)
  75 #endif
  76
  77 #define INVALID_SN_DELTA        0xffff
  78
  79 static unsigned int iscsi_max_lun = 512;
  80 module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
  81
  82 /* global data */
  83 static kmem_cache_t *taskcache;
  84
  85 static inline void
  86 iscsi_buf_init_virt(struct iscsi_buf *ibuf, char *vbuf, int size)
  87 {
  88         sg_init_one(&ibuf->sg, (u8 *)vbuf, size);
  89         ibuf->sent = 0;
  90 }
  91
  92 static inline void
  93 iscsi_buf_init_iov(struct iscsi_buf *ibuf, char *vbuf, int size)
  94 {
  95         ibuf->sg.page = (void*)vbuf;
  96         ibuf->sg.offset = (unsigned int)-1;
  97         ibuf->sg.length = size;
  98         ibuf->sent = 0;
  99 }
 100
 101 static inline void*
 102 iscsi_buf_iov_base(struct iscsi_buf *ibuf)
 103 {
 104         return (char*)ibuf->sg.page + ibuf->sent;
 105 }
 106
 107 static inline void
 108 iscsi_buf_init_sg(struct iscsi_buf *ibuf, struct scatterlist *sg)
 109 {
 110         /*
 111          * Fastpath: sg element fits into single page
 112          */
 113         if (sg->length + sg->offset <= PAGE_SIZE && page_count(sg->page) >= 2) {
 114                 ibuf->sg.page = sg->page;
 115                 ibuf->sg.offset = sg->offset;
 116                 ibuf->sg.length = sg->length;
 117         } else
 118                 iscsi_buf_init_iov(ibuf, page_address(sg->page), sg->length);
 119         ibuf->sent = 0;
 120 }
 121
 122 static inline int
 123 iscsi_buf_left(struct iscsi_buf *ibuf)
 124 {
 125         int rc;
 126
 127         rc = ibuf->sg.length - ibuf->sent;
 128         BUG_ON(rc < 0);
 129         return rc;
 130 }
 131
 132 static inline void
 133 iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf,
 134                  u8* crc)
 135 {
 136         crypto_digest_digest(conn->tx_tfm, &buf->sg, 1, crc);
 137         buf->sg.length += sizeof(uint32_t);
 138 }
 139
 140 static void
 141 iscsi_conn_failure(struct iscsi_conn *conn, enum iscsi_err err)
 142 {
 143         struct iscsi_session *session = conn->session;
 144         unsigned long flags;
 145
 146         spin_lock_irqsave(&session->lock, flags);
 147         if (session->conn_cnt == 1 || session->leadconn == conn)
 148                 session->state = ISCSI_STATE_FAILED;
 149         spin_unlock_irqrestore(&session->lock, flags);
 150         set_bit(SUSPEND_BIT, &conn->suspend_tx);
 151         set_bit(SUSPEND_BIT, &conn->suspend_rx);
 152         iscsi_conn_error(iscsi_handle(conn), err);
 153 }
 154
 155 static inline int
 156 iscsi_check_assign_cmdsn(struct iscsi_session *session, struct iscsi_nopin *hdr)
 157 {
 158         uint32_t max_cmdsn = be32_to_cpu(hdr->max_cmdsn);
 159         uint32_t exp_cmdsn = be32_to_cpu(hdr->exp_cmdsn);
 160
 161         if (max_cmdsn < exp_cmdsn -1 &&
 162             max_cmdsn > exp_cmdsn - INVALID_SN_DELTA)
 163                 return ISCSI_ERR_MAX_CMDSN;
 164         if (max_cmdsn > session->max_cmdsn ||
 165             max_cmdsn < session->max_cmdsn - INVALID_SN_DELTA)
 166                 session->max_cmdsn = max_cmdsn;
 167         if (exp_cmdsn > session->exp_cmdsn ||
 168             exp_cmdsn < session->exp_cmdsn - INVALID_SN_DELTA)
 169                 session->exp_cmdsn = exp_cmdsn;
 170
 171         return 0;
 172 }
 173
 174 static inline int
 175 iscsi_hdr_extract(struct iscsi_conn *conn)
 176 {
 177         struct sk_buff *skb = conn->in.skb;
 178
 179         if (conn->in.copy >= conn->hdr_size &&
 180             conn->in_progress == IN_PROGRESS_WAIT_HEADER) {
 181                 /*
 182                  * Zero-copy PDU Header: using connection context
 183                  * to store header pointer.
 184                  */
 185                 if (skb_shinfo(skb)->frag_list == NULL &&
 186                     !skb_shinfo(skb)->nr_frags)
 187                         conn->in.hdr = (struct iscsi_hdr *)
 188                                 ((char*)skb->data + conn->in.offset);
 189                 else {
 190                         /* ignoring return code since we checked
 191                          * in.copy before */
 192                         skb_copy_bits(skb, conn->in.offset,
 193                                 &conn->hdr, conn->hdr_size);
 194                         conn->in.hdr = &conn->hdr;
 195                 }
 196                 conn->in.offset += conn->hdr_size;
 197                 conn->in.copy -= conn->hdr_size;
 198         } else {
 199                 int hdr_remains;
 200                 int copylen;
 201
 202                 /*
 203                  * PDU header scattered across SKB's,
 204                  * copying it... This'll happen quite rarely.
 205                  */
 206
 207                 if (conn->in_progress == IN_PROGRESS_WAIT_HEADER)
 208                         conn->in.hdr_offset = 0;
 209
 210                 hdr_remains = conn->hdr_size - conn->in.hdr_offset;
 211                 BUG_ON(hdr_remains <= 0);
 212
 213                 copylen = min(conn->in.copy, hdr_remains);
 214                 skb_copy_bits(skb, conn->in.offset,
 215                         (char*)&conn->hdr + conn->in.hdr_offset, copylen);
 216
 217                 debug_tcp("PDU gather offset %d bytes %d in.offset %d "
 218                        "in.copy %d\n", conn->in.hdr_offset, copylen,
 219                        conn->in.offset, conn->in.copy);
 220
 221                 conn->in.offset += copylen;
 222                 conn->in.copy -= copylen;
 223                 if (copylen < hdr_remains)  {
 224                         conn->in_progress = IN_PROGRESS_HEADER_GATHER;
 225                         conn->in.hdr_offset += copylen;
 226                         return -EAGAIN;
 227                 }
 228                 conn->in.hdr = &conn->hdr;
 229                 conn->discontiguous_hdr_cnt++;
 230                 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
 231         }
 232
 233         return 0;
 234 }
 235
 236 static inline void
 237 iscsi_ctask_cleanup(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 238 {
 239         struct scsi_cmnd *sc = ctask->sc;
 240         struct iscsi_session *session = conn->session;
 241
 242         spin_lock(&session->lock);
 243         if (unlikely(!sc)) {
 244                 spin_unlock(&session->lock);
 245                 return;
 246         }
 247         if (sc->sc_data_direction == DMA_TO_DEVICE) {
 248                 struct iscsi_data_task *dtask, *n;
 249                 /* WRITE: cleanup Data-Out's if any */
 250                 spin_lock(&conn->lock);
 251                 list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) {
 252                         list_del(&dtask->item);
 253                         mempool_free(dtask, ctask->datapool);
 254                 }
 255                 spin_unlock(&conn->lock);
 256         }
 257         ctask->xmstate = XMSTATE_IDLE;
 258         ctask->r2t = NULL;
 259         ctask->sc = NULL;
 260         __kfifo_put(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
 261         spin_unlock(&session->lock);
 262 }
 263
 264 /**
 265  * iscsi_cmd_rsp - SCSI Command Response processing
 266  * @conn: iscsi connection
 267  * @ctask: scsi command task
 268  **/
 269 static int
 270 iscsi_cmd_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 271 {
 272         int rc;
 273         struct iscsi_cmd_rsp *rhdr = (struct iscsi_cmd_rsp *)conn->in.hdr;
 274         struct iscsi_session *session = conn->session;
 275         struct scsi_cmnd *sc = ctask->sc;
 276
 277         rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
 278         if (rc) {
 279                 sc->result = (DID_ERROR << 16);
 280                 goto out;
 281         }
 282
 283         conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
 284
 285         sc->result = (DID_OK << 16) | rhdr->cmd_status;
 286
 287         if (rhdr->response != ISCSI_STATUS_CMD_COMPLETED) {
 288                 sc->result = (DID_ERROR << 16);
 289                 goto out;
 290         }
 291
 292         if (rhdr->cmd_status == SAM_STAT_CHECK_CONDITION && conn->senselen) {
 293                 int sensecopy = min(conn->senselen, SCSI_SENSE_BUFFERSIZE);
 294
 295                 memcpy(sc->sense_buffer, conn->data + 2, sensecopy);
 296                 debug_scsi("copied %d bytes of sense\n", sensecopy);
 297         }
 298
 299         if (sc->sc_data_direction == DMA_TO_DEVICE)
 300                 goto out;
 301
 302         if (rhdr->flags & ISCSI_FLAG_CMD_UNDERFLOW) {
 303                 int res_count = be32_to_cpu(rhdr->residual_count);
 304
 305                 if (res_count > 0 && res_count <= sc->request_bufflen)
 306                         sc->resid = res_count;
 307                 else
 308                         sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
 309         } else if (rhdr->flags & ISCSI_FLAG_CMD_BIDI_UNDERFLOW)
 310                 sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
 311         else if (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW)
 312                 sc->resid = be32_to_cpu(rhdr->residual_count);
 313
 314 out:
 315         debug_scsi("done [sc %lx res %d itt 0x%x]\n",
 316                    (long)sc, sc->result, ctask->itt);
 317         conn->scsirsp_pdus_cnt++;
 318         iscsi_ctask_cleanup(conn, ctask);
 319         sc->scsi_done(sc);
 320         return rc;
 321 }
 322
 323 /**
 324  * iscsi_data_rsp - SCSI Data-In Response processing
 325  * @conn: iscsi connection
 326  * @ctask: scsi command task
 327  **/
 328 static int
 329 iscsi_data_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 330 {
 331         int rc;
 332         struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)conn->in.hdr;
 333         struct iscsi_session *session = conn->session;
 334         int datasn = be32_to_cpu(rhdr->datasn);
 335
 336         rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
 337         if (rc)
 338                 return rc;
 339         /*
 340          * setup Data-In byte counter (gets decremented..)
 341          */
 342         ctask->data_count = conn->in.datalen;
 343
 344         if (conn->in.datalen == 0)
 345                 return 0;
 346
 347         if (ctask->datasn != datasn)
 348                 return ISCSI_ERR_DATASN;
 349
 350         ctask->datasn++;
 351
 352         ctask->data_offset = be32_to_cpu(rhdr->offset);
 353         if (ctask->data_offset + conn->in.datalen > ctask->total_length)
 354                 return ISCSI_ERR_DATA_OFFSET;
 355
 356         if (rhdr->flags & ISCSI_FLAG_DATA_STATUS) {
 357                 struct scsi_cmnd *sc = ctask->sc;
 358
 359                 conn->exp_statsn = be32_to_cpu(rhdr->statsn) + 1;
 360                 if (rhdr->flags & ISCSI_FLAG_CMD_UNDERFLOW) {
 361                         int res_count = be32_to_cpu(rhdr->residual_count);
 362
 363                         if (res_count > 0 &&
 364                             res_count <= sc->request_bufflen) {
 365                                 sc->resid = res_count;
 366                                 sc->result = (DID_OK << 16) | rhdr->cmd_status;
 367                         } else
 368                                 sc->result = (DID_BAD_TARGET << 16) |
 369                                         rhdr->cmd_status;
 370                 } else if (rhdr->flags & ISCSI_FLAG_CMD_BIDI_UNDERFLOW)
 371                         sc->result = (DID_BAD_TARGET << 16) | rhdr->cmd_status;
 372                 else if (rhdr->flags & ISCSI_FLAG_CMD_OVERFLOW) {
 373                         sc->resid = be32_to_cpu(rhdr->residual_count);
 374                         sc->result = (DID_OK << 16) | rhdr->cmd_status;
 375                 } else
 376                         sc->result = (DID_OK << 16) | rhdr->cmd_status;
 377         }
 378
 379         conn->datain_pdus_cnt++;
 380         return 0;
 381 }
 382
 383 /**
 384  * iscsi_solicit_data_init - initialize first Data-Out
 385  * @conn: iscsi connection
 386  * @ctask: scsi command task
 387  * @r2t: R2T info
 388  *
 389  * Notes:
 390  *      Initialize first Data-Out within this R2T sequence and finds
 391  *      proper data_offset within this SCSI command.
 392  *
 393  *      This function is called with connection lock taken.
 394  **/
 395 static void
 396 iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
 397                         struct iscsi_r2t_info *r2t)
 398 {
 399         struct iscsi_data *hdr;
 400         struct iscsi_data_task *dtask;
 401         struct scsi_cmnd *sc = ctask->sc;
 402
 403         dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
 404         BUG_ON(!dtask);
 405         hdr = &dtask->hdr;
 406         memset(hdr, 0, sizeof(struct iscsi_data));
 407         hdr->ttt = r2t->ttt;
 408         hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
 409         r2t->solicit_datasn++;
 410         hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
 411         memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
 412         hdr->itt = ctask->hdr.itt;
 413         hdr->exp_statsn = r2t->exp_statsn;
 414         hdr->offset = cpu_to_be32(r2t->data_offset);
 415         if (r2t->data_length > conn->max_xmit_dlength) {
 416                 hton24(hdr->dlength, conn->max_xmit_dlength);
 417                 r2t->data_count = conn->max_xmit_dlength;
 418                 hdr->flags = 0;
 419         } else {
 420                 hton24(hdr->dlength, r2t->data_length);
 421                 r2t->data_count = r2t->data_length;
 422                 hdr->flags = ISCSI_FLAG_CMD_FINAL;
 423         }
 424         conn->dataout_pdus_cnt++;
 425
 426         r2t->sent = 0;
 427
 428         iscsi_buf_init_virt(&r2t->headbuf, (char*)hdr,
 429                            sizeof(struct iscsi_hdr));
 430
 431         r2t->dtask = dtask;
 432
 433         if (sc->use_sg) {
 434                 int i, sg_count = 0;
 435                 struct scatterlist *sg = sc->request_buffer;
 436
 437                 r2t->sg = NULL;
 438                 for (i = 0; i < sc->use_sg; i++, sg += 1) {
 439                         /* FIXME: prefetch ? */
 440                         if (sg_count + sg->length > r2t->data_offset) {
 441                                 int page_offset;
 442
 443                                 /* sg page found! */
 444
 445                                 /* offset within this page */
 446                                 page_offset = r2t->data_offset - sg_count;
 447
 448                                 /* fill in this buffer */
 449                                 iscsi_buf_init_sg(&r2t->sendbuf, sg);
 450                                 r2t->sendbuf.sg.offset += page_offset;
 451                                 r2t->sendbuf.sg.length -= page_offset;
 452
 453                                 /* xmit logic will continue with next one */
 454                                 r2t->sg = sg + 1;
 455                                 break;
 456                         }
 457                         sg_count += sg->length;
 458                 }
 459                 BUG_ON(r2t->sg == NULL);
 460         } else
 461                 iscsi_buf_init_iov(&ctask->sendbuf,
 462                             (char*)sc->request_buffer + r2t->data_offset,
 463                             r2t->data_count);
 464
 465         list_add(&dtask->item, &ctask->dataqueue);
 466 }
 467
 468 /**
 469  * iscsi_r2t_rsp - iSCSI R2T Response processing
 470  * @conn: iscsi connection
 471  * @ctask: scsi command task
 472  **/
 473 static int
 474 iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 475 {
 476         struct iscsi_r2t_info *r2t;
 477         struct iscsi_session *session = conn->session;
 478         struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)conn->in.hdr;
 479         int r2tsn = be32_to_cpu(rhdr->r2tsn);
 480         int rc;
 481
 482         if (conn->in.ahslen)
 483                 return ISCSI_ERR_AHSLEN;
 484
 485         if (conn->in.datalen)
 486                 return ISCSI_ERR_DATALEN;
 487
 488         if (ctask->exp_r2tsn && ctask->exp_r2tsn != r2tsn)
 489                 return ISCSI_ERR_R2TSN;
 490
 491         rc = iscsi_check_assign_cmdsn(session, (struct iscsi_nopin*)rhdr);
 492         if (rc)
 493                 return rc;
 494
 495         /* FIXME: use R2TSN to detect missing R2T */
 496
 497         /* fill-in new R2T associated with the task */
 498         spin_lock(&session->lock);
 499         if (!ctask->sc || ctask->mtask ||
 500              session->state != ISCSI_STATE_LOGGED_IN) {
 501                 printk(KERN_INFO "iscsi_tcp: dropping R2T itt %d in "
 502                        "recovery...\n", ctask->itt);
 503                 spin_unlock(&session->lock);
 504                 return 0;
 505         }
 506         rc = __kfifo_get(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
 507         BUG_ON(!rc);
 508
 509         r2t->exp_statsn = rhdr->statsn;
 510         r2t->data_length = be32_to_cpu(rhdr->data_length);
 511         if (r2t->data_length == 0 ||
 512             r2t->data_length > session->max_burst) {
 513                 spin_unlock(&session->lock);
 514                 return ISCSI_ERR_DATALEN;
 515         }
 516
 517         r2t->data_offset = be32_to_cpu(rhdr->data_offset);
 518         if (r2t->data_offset + r2t->data_length > ctask->total_length) {
 519                 spin_unlock(&session->lock);
 520                 return ISCSI_ERR_DATALEN;
 521         }
 522
 523         r2t->ttt = rhdr->ttt; /* no flip */
 524         r2t->solicit_datasn = 0;
 525
 526         iscsi_solicit_data_init(conn, ctask, r2t);
 527
 528         ctask->exp_r2tsn = r2tsn + 1;
 529         ctask->xmstate |= XMSTATE_SOL_HDR;
 530         __kfifo_put(ctask->r2tqueue, (void*)&r2t, sizeof(void*));
 531         __kfifo_put(conn->writequeue, (void*)&ctask, sizeof(void*));
 532
 533         schedule_work(&conn->xmitwork);
 534         conn->r2t_pdus_cnt++;
 535         spin_unlock(&session->lock);
 536
 537         return 0;
 538 }
 539
 540 static int
 541 iscsi_hdr_recv(struct iscsi_conn *conn)
 542 {
 543         int rc = 0;
 544         struct iscsi_hdr *hdr;
 545         struct iscsi_cmd_task *ctask;
 546         struct iscsi_session *session = conn->session;
 547         uint32_t cdgst, rdgst = 0;
 548
 549         hdr = conn->in.hdr;
 550
 551         /* verify PDU length */
 552         conn->in.datalen = ntoh24(hdr->dlength);
 553         if (conn->in.datalen > conn->max_recv_dlength) {
 554                 printk(KERN_ERR "iscsi_tcp: datalen %d > %d\n",
 555                        conn->in.datalen, conn->max_recv_dlength);
 556                 return ISCSI_ERR_DATALEN;
 557         }
 558         conn->data_copied = 0;
 559
 560         /* read AHS */
 561         conn->in.ahslen = hdr->hlength * 4;
 562         conn->in.offset += conn->in.ahslen;
 563         conn->in.copy -= conn->in.ahslen;
 564         if (conn->in.copy < 0) {
 565                 printk(KERN_ERR "iscsi_tcp: can't handle AHS with length "
 566                        "%d bytes\n", conn->in.ahslen);
 567                 return ISCSI_ERR_AHSLEN;
 568         }
 569
 570         /* calculate read padding */
 571         conn->in.padding = conn->in.datalen & (ISCSI_PAD_LEN-1);
 572         if (conn->in.padding) {
 573                 conn->in.padding = ISCSI_PAD_LEN - conn->in.padding;
 574                 debug_scsi("read padding %d bytes\n", conn->in.padding);
 575         }
 576
 577         if (conn->hdrdgst_en) {
 578                 struct scatterlist sg;
 579
 580                 sg_init_one(&sg, (u8 *)hdr,
 581                             sizeof(struct iscsi_hdr) + conn->in.ahslen);
 582                 crypto_digest_digest(conn->rx_tfm, &sg, 1, (u8 *)&cdgst);
 583                 rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) +
 584                                      conn->in.ahslen);
 585                 if (cdgst != rdgst) {
 586                         printk(KERN_ERR "iscsi_tcp: itt %x: hdrdgst error "
 587                                "recv 0x%x calc 0x%x\n", conn->in.itt, rdgst,
 588                                cdgst);
 589                         return ISCSI_ERR_HDR_DGST;
 590                 }
 591         }
 592
 593         /* save opcode for later */
 594         conn->in.opcode = hdr->opcode & ISCSI_OPCODE_MASK;
 595
 596         /* verify itt (itt encoding: age+cid+itt) */
 597         if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
 598                 if ((hdr->itt & AGE_MASK) !=
 599                                 (session->age << AGE_SHIFT)) {
 600                         printk(KERN_ERR "iscsi_tcp: received itt %x expected "
 601                                 "session age (%x)\n", hdr->itt,
 602                                 session->age & AGE_MASK);
 603                         return ISCSI_ERR_BAD_ITT;
 604                 }
 605
 606                 if ((hdr->itt & CID_MASK) != (conn->id << CID_SHIFT)) {
 607                         printk(KERN_ERR "iscsi_tcp: received itt %x, expected "
 608                                 "CID (%x)\n", hdr->itt, conn->id);
 609                         return ISCSI_ERR_BAD_ITT;
 610                 }
 611                 conn->in.itt = hdr->itt & ITT_MASK;
 612         } else
 613                 conn->in.itt = hdr->itt;
 614
 615         debug_tcp("opcode 0x%x offset %d copy %d ahslen %d datalen %d\n",
 616                   hdr->opcode, conn->in.offset, conn->in.copy,
 617                   conn->in.ahslen, conn->in.datalen);
 618
 619         if (conn->in.itt < session->cmds_max) {
 620                 ctask = (struct iscsi_cmd_task *)session->cmds[conn->in.itt];
 621
 622                 if (!ctask->sc) {
 623                         printk(KERN_INFO "iscsi_tcp: dropping ctask with "
 624                                "itt 0x%x\n", ctask->itt);
 625                         conn->in.datalen = 0; /* force drop */
 626                         return 0;
 627                 }
 628
 629                 if (ctask->sc->SCp.phase != session->age) {
 630                         printk(KERN_ERR "iscsi_tcp: ctask's session age %d, "
 631                                 "expected %d\n", ctask->sc->SCp.phase,
 632                                 session->age);
 633                         return ISCSI_ERR_SESSION_FAILED;
 634                 }
 635
 636                 conn->in.ctask = ctask;
 637
 638                 debug_scsi("rsp [op 0x%x cid %d sc %lx itt 0x%x len %d]\n",
 639                            hdr->opcode, conn->id, (long)ctask->sc,
 640                            ctask->itt, conn->in.datalen);
 641
 642                 switch(conn->in.opcode) {
 643                 case ISCSI_OP_SCSI_CMD_RSP:
 644                         BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
 645                         if (!conn->in.datalen)
 646                                 rc = iscsi_cmd_rsp(conn, ctask);
 647                         else
 648                                 /*
 649                                  * got sense or response data; copying PDU
 650                                  * Header to the connection's header
 651                                  * placeholder
 652                                  */
 653                                 memcpy(&conn->hdr, hdr,
 654                                        sizeof(struct iscsi_hdr));
 655                         break;
 656                 case ISCSI_OP_SCSI_DATA_IN:
 657                         BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
 658                         /* save flags for non-exceptional status */
 659                         conn->in.flags = hdr->flags;
 660                         /* save cmd_status for sense data */
 661                         conn->in.cmd_status =
 662                                 ((struct iscsi_data_rsp*)hdr)->cmd_status;
 663                         rc = iscsi_data_rsp(conn, ctask);
 664                         break;
 665                 case ISCSI_OP_R2T:
 666                         BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
 667                         if (ctask->sc->sc_data_direction == DMA_TO_DEVICE)
 668                                 rc = iscsi_r2t_rsp(conn, ctask);
 669                         else
 670                                 rc = ISCSI_ERR_PROTO;
 671                         break;
 672                 default:
 673                         rc = ISCSI_ERR_BAD_OPCODE;
 674                         break;
 675                 }
 676         } else if (conn->in.itt >= ISCSI_MGMT_ITT_OFFSET &&
 677                    conn->in.itt < ISCSI_MGMT_ITT_OFFSET +
 678                                         session->mgmtpool_max) {
 679                 struct iscsi_mgmt_task *mtask = (struct iscsi_mgmt_task *)
 680                                         session->mgmt_cmds[conn->in.itt -
 681                                                 ISCSI_MGMT_ITT_OFFSET];
 682
 683                 debug_scsi("immrsp [op 0x%x cid %d itt 0x%x len %d]\n",
 684                            conn->in.opcode, conn->id, mtask->itt,
 685                            conn->in.datalen);
 686
 687                 switch(conn->in.opcode) {
 688                 case ISCSI_OP_LOGIN_RSP:
 689                 case ISCSI_OP_TEXT_RSP:
 690                 case ISCSI_OP_LOGOUT_RSP:
 691                         rc = iscsi_check_assign_cmdsn(session,
 692                                                  (struct iscsi_nopin*)hdr);
 693                         if (rc)
 694                                 break;
 695
 696                         if (!conn->in.datalen) {
 697                                 rc = iscsi_recv_pdu(iscsi_handle(conn), hdr,
 698                                                     NULL, 0);
 699                                 if (conn->login_mtask != mtask) {
 700                                         spin_lock(&session->lock);
 701                                         __kfifo_put(session->mgmtpool.queue,
 702                                             (void*)&mtask, sizeof(void*));
 703                                         spin_unlock(&session->lock);
 704                                 }
 705                         }
 706                         break;
 707                 case ISCSI_OP_SCSI_TMFUNC_RSP:
 708                         rc = iscsi_check_assign_cmdsn(session,
 709                                                  (struct iscsi_nopin*)hdr);
 710                         if (rc)
 711                                 break;
 712
 713                         if (conn->in.datalen || conn->in.ahslen) {
 714                                 rc = ISCSI_ERR_PROTO;
 715                                 break;
 716                         }
 717                         conn->tmfrsp_pdus_cnt++;
 718                         spin_lock(&session->lock);
 719                         if (conn->tmabort_state == TMABORT_INITIAL) {
 720                                 __kfifo_put(session->mgmtpool.queue,
 721                                                 (void*)&mtask, sizeof(void*));
 722                                 conn->tmabort_state =
 723                                         ((struct iscsi_tm_rsp *)hdr)->
 724                                         response == ISCSI_TMF_RSP_COMPLETE ?
 725                                                 TMABORT_SUCCESS:TMABORT_FAILED;
 726                                 /* unblock eh_abort() */
 727                                 wake_up(&conn->ehwait);
 728                         }
 729                         spin_unlock(&session->lock);
 730                         break;
 731                 case ISCSI_OP_NOOP_IN:
 732                         if (hdr->ttt != ISCSI_RESERVED_TAG) {
 733                                 rc = ISCSI_ERR_PROTO;
 734                                 break;
 735                         }
 736                         rc = iscsi_check_assign_cmdsn(session,
 737                                                 (struct iscsi_nopin*)hdr);
 738                         if (rc)
 739                                 break;
 740                         conn->exp_statsn = be32_to_cpu(hdr->statsn) + 1;
 741
 742                         if (!conn->in.datalen) {
 743                                 struct iscsi_mgmt_task *mtask;
 744
 745                                 rc = iscsi_recv_pdu(iscsi_handle(conn), hdr,
 746                                                     NULL, 0);
 747                                 mtask = (struct iscsi_mgmt_task *)
 748                                         session->mgmt_cmds[conn->in.itt -
 749                                                         ISCSI_MGMT_ITT_OFFSET];
 750                                 if (conn->login_mtask != mtask) {
 751                                         spin_lock(&session->lock);
 752                                         __kfifo_put(session->mgmtpool.queue,
 753                                                   (void*)&mtask, sizeof(void*));
 754                                         spin_unlock(&session->lock);
 755                                 }
 756                         }
 757                         break;
 758                 default:
 759                         rc = ISCSI_ERR_BAD_OPCODE;
 760                         break;
 761                 }
 762         } else if (conn->in.itt == ISCSI_RESERVED_TAG) {
 763                 switch(conn->in.opcode) {
 764                 case ISCSI_OP_NOOP_IN:
 765                         if (!conn->in.datalen) {
 766                                 rc = iscsi_check_assign_cmdsn(session,
 767                                                  (struct iscsi_nopin*)hdr);
 768                                 if (!rc && hdr->ttt != ISCSI_RESERVED_TAG)
 769                                         rc = iscsi_recv_pdu(iscsi_handle(conn),
 770                                                             hdr, NULL, 0);
 771                         } else
 772                                 rc = ISCSI_ERR_PROTO;
 773                         break;
 774                 case ISCSI_OP_REJECT:
 775                         /* we need sth like iscsi_reject_rsp()*/
 776                 case ISCSI_OP_ASYNC_EVENT:
 777                         /* we need sth like iscsi_async_event_rsp() */
 778                         rc = ISCSI_ERR_BAD_OPCODE;
 779                         break;
 780                 default:
 781                         rc = ISCSI_ERR_BAD_OPCODE;
 782                         break;
 783                 }
 784         } else
 785                 rc = ISCSI_ERR_BAD_ITT;
 786
 787         return rc;
 788 }
 789
 790 /**
 791  * iscsi_ctask_copy - copy skb bits to the destanation cmd task
 792  * @conn: iscsi connection
 793  * @ctask: scsi command task
 794  * @buf: buffer to copy to
 795  * @buf_size: size of buffer
 796  * @offset: offset within the buffer
 797  *
 798  * Notes:
 799  *      The function calls skb_copy_bits() and updates per-connection and
 800  *      per-cmd byte counters.
 801  *
 802  *      Read counters (in bytes):
 803  *
 804  *      conn->in.offset         offset within in progress SKB
 805  *      conn->in.copy           left to copy from in progress SKB
 806  *                              including padding
 807  *      conn->in.copied         copied already from in progress SKB
 808  *      conn->data_copied       copied already from in progress buffer
 809  *      ctask->sent             total bytes sent up to the MidLayer
 810  *      ctask->data_count       left to copy from in progress Data-In
 811  *      buf_left                left to copy from in progress buffer
 812  **/
 813 static inline int
 814 iscsi_ctask_copy(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
 815                 void *buf, int buf_size, int offset)
 816 {
 817         int buf_left = buf_size - (conn->data_copied + offset);
 818         int size = min(conn->in.copy, buf_left);
 819         int rc;
 820
 821         size = min(size, ctask->data_count);
 822
 823         debug_tcp("ctask_copy %d bytes at offset %d copied %d\n",
 824                size, conn->in.offset, conn->in.copied);
 825
 826         BUG_ON(size <= 0);
 827         BUG_ON(ctask->sent + size > ctask->total_length);
 828
 829         rc = skb_copy_bits(conn->in.skb, conn->in.offset,
 830                            (char*)buf + (offset + conn->data_copied), size);
 831         /* must fit into skb->len */
 832         BUG_ON(rc);
 833
 834         conn->in.offset += size;
 835         conn->in.copy -= size;
 836         conn->in.copied += size;
 837         conn->data_copied += size;
 838         ctask->sent += size;
 839         ctask->data_count -= size;
 840
 841         BUG_ON(conn->in.copy < 0);
 842         BUG_ON(ctask->data_count < 0);
 843
 844         if (buf_size != (conn->data_copied + offset)) {
 845                 if (!ctask->data_count) {
 846                         BUG_ON(buf_size - conn->data_copied < 0);
 847                         /* done with this PDU */
 848                         return buf_size - conn->data_copied;
 849                 }
 850                 return -EAGAIN;
 851         }
 852
 853         /* done with this buffer or with both - PDU and buffer */
 854         conn->data_copied = 0;
 855         return 0;
 856 }
 857
 858 /**
 859  * iscsi_tcp_copy - copy skb bits to the destanation buffer
 860  * @conn: iscsi connection
 861  * @buf: buffer to copy to
 862  * @buf_size: number of bytes to copy
 863  *
 864  * Notes:
 865  *      The function calls skb_copy_bits() and updates per-connection
 866  *      byte counters.
 867  **/
 868 static inline int
 869 iscsi_tcp_copy(struct iscsi_conn *conn, void *buf, int buf_size)
 870 {
 871         int buf_left = buf_size - conn->data_copied;
 872         int size = min(conn->in.copy, buf_left);
 873         int rc;
 874
 875         debug_tcp("tcp_copy %d bytes at offset %d copied %d\n",
 876                size, conn->in.offset, conn->data_copied);
 877         BUG_ON(size <= 0);
 878
 879         rc = skb_copy_bits(conn->in.skb, conn->in.offset,
 880                            (char*)buf + conn->data_copied, size);
 881         BUG_ON(rc);
 882
 883         conn->in.offset += size;
 884         conn->in.copy -= size;
 885         conn->in.copied += size;
 886         conn->data_copied += size;
 887
 888         if (buf_size != conn->data_copied)
 889                 return -EAGAIN;
 890
 891         return 0;
 892 }
 893
 894 static inline void
 895 partial_sg_digest_update(struct iscsi_conn *conn, struct scatterlist *sg,
 896                          int offset, int length)
 897 {
 898         struct scatterlist temp;
 899
 900         memcpy(&temp, sg, sizeof(struct scatterlist));
 901         temp.offset = offset;
 902         temp.length = length;
 903         crypto_digest_update(conn->data_rx_tfm, &temp, 1);
 904 }
 905
 906 static void
 907 iscsi_recv_digest_update(struct iscsi_conn *conn, char* buf, int len)
 908 {
 909         struct scatterlist tmp;
 910
 911         sg_init_one(&tmp, buf, len);
 912         crypto_digest_update(conn->data_rx_tfm, &tmp, 1);
 913 }
 914
 915 static int iscsi_scsi_data_in(struct iscsi_conn *conn)
 916 {
 917         struct iscsi_cmd_task *ctask = conn->in.ctask;
 918         struct scsi_cmnd *sc = ctask->sc;
 919         struct scatterlist *sg;
 920         int i, offset, rc = 0;
 921
 922         BUG_ON((void*)ctask != sc->SCp.ptr);
 923
 924         /*
 925          * copying Data-In into the Scsi_Cmnd
 926          */
 927         if (!sc->use_sg) {
 928                 i = ctask->data_count;
 929                 rc = iscsi_ctask_copy(conn, ctask, sc->request_buffer,
 930                                       sc->request_bufflen, ctask->data_offset);
 931                 if (rc == -EAGAIN)
 932                         return rc;
 933                 if (conn->datadgst_en)
 934                         iscsi_recv_digest_update(conn, sc->request_buffer, i);
 935                 rc = 0;
 936                 goto done;
 937         }
 938
 939         offset = ctask->data_offset;
 940         sg = sc->request_buffer;
 941
 942         if (ctask->data_offset)
 943                 for (i = 0; i < ctask->sg_count; i++)
 944                         offset -= sg[i].length;
 945         /* we've passed through partial sg*/
 946         if (offset < 0)
 947                 offset = 0;
 948
 949         for (i = ctask->sg_count; i < sc->use_sg; i++) {
 950                 char *dest;
 951
 952                 dest = kmap_atomic(sg[i].page, KM_SOFTIRQ0);
 953                 rc = iscsi_ctask_copy(conn, ctask, dest + sg[i].offset,
 954                                       sg[i].length, offset);
 955                 kunmap_atomic(dest, KM_SOFTIRQ0);
 956                 if (rc == -EAGAIN)
 957                         /* continue with the next SKB/PDU */
 958                         return rc;
 959                 if (!rc) {
 960                         if (conn->datadgst_en) {
 961                                 if (!offset)
 962                                         crypto_digest_update(conn->data_rx_tfm,
 963                                                              &sg[i], 1);
 964                                 else
 965                                         partial_sg_digest_update(conn, &sg[i],
 966                                                         sg[i].offset + offset,
 967                                                         sg[i].length - offset);
 968                         }
 969                         offset = 0;
 970                         ctask->sg_count++;
 971                 }
 972
 973                 if (!ctask->data_count) {
 974                         if (rc && conn->datadgst_en)
 975                                 /*
 976                                  * data-in is complete, but buffer not...
 977                                  */
 978                                 partial_sg_digest_update(conn, &sg[i],
 979                                                 sg[i].offset, sg[i].length-rc);
 980                         rc = 0;
 981                         break;
 982                 }
 983
 984                 if (!conn->in.copy)
 985                         return -EAGAIN;
 986         }
 987         BUG_ON(ctask->data_count);
 988
 989 done:
 990         /* check for non-exceptional status */
 991         if (conn->in.flags & ISCSI_FLAG_DATA_STATUS) {
 992                 debug_scsi("done [sc %lx res %d itt 0x%x]\n",
 993                            (long)sc, sc->result, ctask->itt);
 994                 conn->scsirsp_pdus_cnt++;
 995                 iscsi_ctask_cleanup(conn, ctask);
 996                 sc->scsi_done(sc);
 997         }
 998
 999         return rc;
1000 }
1001
1002 static int
1003 iscsi_data_recv(struct iscsi_conn *conn)
1004 {
1005         struct iscsi_session *session = conn->session;
1006         int rc = 0;
1007
1008         switch(conn->in.opcode) {
1009         case ISCSI_OP_SCSI_DATA_IN:
1010                 rc = iscsi_scsi_data_in(conn);
1011                 break;
1012         case ISCSI_OP_SCSI_CMD_RSP: {
1013                 /*
1014                  * SCSI Sense Data:
1015                  * copying the entire Data Segment.
1016                  */
1017                 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) {
1018                         rc = -EAGAIN;
1019                         goto exit;
1020                 }
1021
1022                 /*
1023                  * check for sense
1024                  */
1025                 conn->in.hdr = &conn->hdr;
1026                 conn->senselen = (conn->data[0] << 8) | conn->data[1];
1027                 rc = iscsi_cmd_rsp(conn, conn->in.ctask);
1028                 if (!rc && conn->datadgst_en)
1029                         iscsi_recv_digest_update(conn, conn->data,
1030                                                  conn->in.datalen);
1031         }
1032         break;
1033         case ISCSI_OP_TEXT_RSP:
1034         case ISCSI_OP_LOGIN_RSP:
1035         case ISCSI_OP_NOOP_IN: {
1036                 struct iscsi_mgmt_task *mtask = NULL;
1037
1038                 if (conn->in.itt != ISCSI_RESERVED_TAG)
1039                         mtask = (struct iscsi_mgmt_task *)
1040                                 session->mgmt_cmds[conn->in.itt -
1041                                         ISCSI_MGMT_ITT_OFFSET];
1042
1043                 /*
1044                  * Collect data segment to the connection's data
1045                  * placeholder
1046                  */
1047                 if (iscsi_tcp_copy(conn, conn->data, conn->in.datalen)) {
1048                         rc = -EAGAIN;
1049                         goto exit;
1050                 }
1051
1052                 rc = iscsi_recv_pdu(iscsi_handle(conn), conn->in.hdr,
1053                                     conn->data, conn->in.datalen);
1054
1055                 if (!rc && conn->datadgst_en &&
1056                         conn->in.opcode != ISCSI_OP_LOGIN_RSP)
1057                         iscsi_recv_digest_update(conn, conn->data,
1058                                                 conn->in.datalen);
1059
1060                 if (mtask && conn->login_mtask != mtask) {
1061                         spin_lock(&session->lock);
1062                         __kfifo_put(session->mgmtpool.queue, (void*)&mtask,
1063                                     sizeof(void*));
1064                         spin_unlock(&session->lock);
1065                 }
1066         }
1067         break;
1068         case ISCSI_OP_ASYNC_EVENT:
1069         case ISCSI_OP_REJECT:
1070         default:
1071                 BUG_ON(1);
1072         }
1073 exit:
1074         return rc;
1075 }
1076
1077 /**
1078  * iscsi_tcp_data_recv - TCP receive in sendfile fashion
1079  * @rd_desc: read descriptor
1080  * @skb: socket buffer
1081  * @offset: offset in skb
1082  * @len: skb->len - offset
1083  **/
1084 static int
1085 iscsi_tcp_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
1086                 unsigned int offset, size_t len)
1087 {
1088         int rc;
1089         struct iscsi_conn *conn = rd_desc->arg.data;
1090         int processed;
1091         char pad[ISCSI_PAD_LEN];
1092         struct scatterlist sg;
1093
1094         /*
1095          * Save current SKB and its offset in the corresponding
1096          * connection context.
1097          */
1098         conn->in.copy = skb->len - offset;
1099         conn->in.offset = offset;
1100         conn->in.skb = skb;
1101         conn->in.len = conn->in.copy;
1102         BUG_ON(conn->in.copy <= 0);
1103         debug_tcp("in %d bytes\n", conn->in.copy);
1104
1105 more:
1106         conn->in.copied = 0;
1107         rc = 0;
1108
1109         if (unlikely(conn->suspend_rx)) {
1110                 debug_tcp("conn %d Rx suspended!\n", conn->id);
1111                 return 0;
1112         }
1113
1114         if (conn->in_progress == IN_PROGRESS_WAIT_HEADER ||
1115             conn->in_progress == IN_PROGRESS_HEADER_GATHER) {
1116                 rc = iscsi_hdr_extract(conn);
1117                 if (rc) {
1118                        if (rc == -EAGAIN)
1119                                 goto nomore;
1120                        else {
1121                                 iscsi_conn_failure(conn, rc);
1122                                 return 0;
1123                        }
1124                 }
1125
1126                 /*
1127                  * Verify and process incoming PDU header.
1128                  */
1129                 rc = iscsi_hdr_recv(conn);
1130                 if (!rc && conn->in.datalen) {
1131                         if (conn->datadgst_en) {
1132                                 BUG_ON(!conn->data_rx_tfm);
1133                                 crypto_digest_init(conn->data_rx_tfm);
1134                         }
1135                         conn->in_progress = IN_PROGRESS_DATA_RECV;
1136                 } else if (rc) {
1137                         iscsi_conn_failure(conn, rc);
1138                         return 0;
1139                 }
1140         }
1141
1142         if (conn->in_progress == IN_PROGRESS_DDIGEST_RECV) {
1143                 uint32_t recv_digest;
1144                 debug_tcp("extra data_recv offset %d copy %d\n",
1145                           conn->in.offset, conn->in.copy);
1146                 skb_copy_bits(conn->in.skb, conn->in.offset,
1147                                 &recv_digest, 4);
1148                 conn->in.offset += 4;
1149                 conn->in.copy -= 4;
1150                 if (recv_digest != conn->in.datadgst) {
1151                         debug_tcp("iscsi_tcp: data digest error!"
1152                                   "0x%x != 0x%x\n", recv_digest,
1153                                   conn->in.datadgst);
1154                         iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST);
1155                         return 0;
1156                 } else {
1157                         debug_tcp("iscsi_tcp: data digest match!"
1158                                   "0x%x == 0x%x\n", recv_digest,
1159                                   conn->in.datadgst);
1160                         conn->in_progress = IN_PROGRESS_WAIT_HEADER;
1161                 }
1162         }
1163
1164         if (conn->in_progress == IN_PROGRESS_DATA_RECV && conn->in.copy) {
1165
1166                 debug_tcp("data_recv offset %d copy %d\n",
1167                        conn->in.offset, conn->in.copy);
1168
1169                 rc = iscsi_data_recv(conn);
1170                 if (rc) {
1171                         if (rc == -EAGAIN) {
1172                                 rd_desc->count = conn->in.datalen -
1173                                                 conn->in.ctask->data_count;
1174                                 goto again;
1175                         }
1176                         iscsi_conn_failure(conn, rc);
1177                         return 0;
1178                 }
1179                 conn->in.copy -= conn->in.padding;
1180                 conn->in.offset += conn->in.padding;
1181                 if (conn->datadgst_en) {
1182                         if (conn->in.padding) {
1183                                 debug_tcp("padding -> %d\n", conn->in.padding);
1184                                 memset(pad, 0, conn->in.padding);
1185                                 sg_init_one(&sg, pad, conn->in.padding);
1186                                 crypto_digest_update(conn->data_rx_tfm, &sg, 1);
1187                         }
1188                         crypto_digest_final(conn->data_rx_tfm,
1189                                             (u8 *) & conn->in.datadgst);
1190                         debug_tcp("rx digest 0x%x\n", conn->in.datadgst);
1191                         conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
1192                 } else
1193                         conn->in_progress = IN_PROGRESS_WAIT_HEADER;
1194         }
1195
1196         debug_tcp("f, processed %d from out of %d padding %d\n",
1197                conn->in.offset - offset, (int)len, conn->in.padding);
1198         BUG_ON(conn->in.offset - offset > len);
1199
1200         if (conn->in.offset - offset != len) {
1201                 debug_tcp("continue to process %d bytes\n",
1202                        (int)len - (conn->in.offset - offset));
1203                 goto more;
1204         }
1205
1206 nomore:
1207         processed = conn->in.offset - offset;
1208         BUG_ON(processed == 0);
1209         return processed;
1210
1211 again:
1212         processed = conn->in.offset - offset;
1213         debug_tcp("c, processed %d from out of %d rd_desc_cnt %d\n",
1214                   processed, (int)len, (int)rd_desc->count);
1215         BUG_ON(processed == 0);
1216         BUG_ON(processed > len);
1217
1218         conn->rxdata_octets += processed;
1219         return processed;
1220 }
1221
1222 static void
1223 iscsi_tcp_data_ready(struct sock *sk, int flag)
1224 {
1225         struct iscsi_conn *conn = sk->sk_user_data;
1226         read_descriptor_t rd_desc;
1227
1228         read_lock(&sk->sk_callback_lock);
1229
1230         /* use rd_desc to pass 'conn' to iscsi_tcp_data_recv */
1231         rd_desc.arg.data = conn;
1232         rd_desc.count = 0;
1233         tcp_read_sock(sk, &rd_desc, iscsi_tcp_data_recv);
1234
1235         read_unlock(&sk->sk_callback_lock);
1236 }
1237
1238 static void
1239 iscsi_tcp_state_change(struct sock *sk)
1240 {
1241         struct iscsi_conn *conn;
1242         struct iscsi_session *session;
1243         void (*old_state_change)(struct sock *);
1244
1245         read_lock(&sk->sk_callback_lock);
1246
1247         conn = (struct iscsi_conn*)sk->sk_user_data;
1248         session = conn->session;
1249
1250         if ((sk->sk_state == TCP_CLOSE_WAIT ||
1251              sk->sk_state == TCP_CLOSE) &&
1252             !atomic_read(&sk->sk_rmem_alloc)) {
1253                 debug_tcp("iscsi_tcp_state_change: TCP_CLOSE|TCP_CLOSE_WAIT\n");
1254                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1255         }
1256
1257         old_state_change = conn->old_state_change;
1258
1259         read_unlock(&sk->sk_callback_lock);
1260
1261         old_state_change(sk);
1262 }
1263
1264 /**
1265  * iscsi_write_space - Called when more output buffer space is available
1266  * @sk: socket space is available for
1267  **/
1268 static void
1269 iscsi_write_space(struct sock *sk)
1270 {
1271         struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data;
1272         conn->old_write_space(sk);
1273         debug_tcp("iscsi_write_space: cid %d\n", conn->id);
1274         clear_bit(SUSPEND_BIT, &conn->suspend_tx);
1275         schedule_work(&conn->xmitwork);
1276 }
1277
1278 static void
1279 iscsi_conn_set_callbacks(struct iscsi_conn *conn)
1280 {
1281         struct sock *sk = conn->sock->sk;
1282
1283         /* assign new callbacks */
1284         write_lock_bh(&sk->sk_callback_lock);
1285         sk->sk_user_data = conn;
1286         conn->old_data_ready = sk->sk_data_ready;
1287         conn->old_state_change = sk->sk_state_change;
1288         conn->old_write_space = sk->sk_write_space;
1289         sk->sk_data_ready = iscsi_tcp_data_ready;
1290         sk->sk_state_change = iscsi_tcp_state_change;
1291         sk->sk_write_space = iscsi_write_space;
1292         write_unlock_bh(&sk->sk_callback_lock);
1293 }
1294
1295 static void
1296 iscsi_conn_restore_callbacks(struct iscsi_conn *conn)
1297 {
1298         struct sock *sk = conn->sock->sk;
1299
1300         /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
1301         write_lock_bh(&sk->sk_callback_lock);
1302         sk->sk_user_data    = NULL;
1303         sk->sk_data_ready   = conn->old_data_ready;
1304         sk->sk_state_change = conn->old_state_change;
1305         sk->sk_write_space  = conn->old_write_space;
1306         sk->sk_no_check  = 0;
1307         write_unlock_bh(&sk->sk_callback_lock);
1308 }
1309
1310 /**
1311  * iscsi_send - generic send routine
1312  * @sk: kernel's socket
1313  * @buf: buffer to write from
1314  * @size: actual size to write
1315  * @flags: socket's flags
1316  *
1317  * Notes:
1318  *      depending on buffer will use tcp_sendpage() or tcp_sendmsg().
1319  *      buf->sg.offset == -1 tells us that buffer is non S/G and forces
1320  *      to use tcp_sendmsg().
1321  */
1322 static inline int
1323 iscsi_send(struct socket *sk, struct iscsi_buf *buf, int size, int flags)
1324 {
1325         int res;
1326
1327         if ((int)buf->sg.offset >= 0) {
1328                 int offset = buf->sg.offset + buf->sent;
1329
1330                 /* tcp_sendpage */
1331                 res = sk->ops->sendpage(sk, buf->sg.page, offset, size, flags);
1332         } else {
1333                 struct msghdr msg;
1334
1335                 buf->iov.iov_base = iscsi_buf_iov_base(buf);
1336                 buf->iov.iov_len = size;
1337
1338                 memset(&msg, 0, sizeof(struct msghdr));
1339
1340                 /* tcp_sendmsg */
1341                 res = kernel_sendmsg(sk, &msg, &buf->iov, 1, size);
1342         }
1343
1344         return res;
1345 }
1346
1347 /**
1348  * iscsi_sendhdr - send PDU Header via tcp_sendpage()
1349  * @conn: iscsi connection
1350  * @buf: buffer to write from
1351  * @datalen: lenght of data to be sent after the header
1352  *
1353  * Notes:
1354  *      (Tx, Fast Path)
1355  **/
1356 static inline int
1357 iscsi_sendhdr(struct iscsi_conn *conn, struct iscsi_buf *buf, int datalen)
1358 {
1359         struct socket *sk = conn->sock;
1360         int flags = 0; /* MSG_DONTWAIT; */
1361         int res, size;
1362
1363         size = buf->sg.length - buf->sent;
1364         BUG_ON(buf->sent + size > buf->sg.length);
1365         if (buf->sent + size != buf->sg.length || datalen)
1366                 flags |= MSG_MORE;
1367
1368         res = iscsi_send(sk, buf, size, flags);
1369         debug_tcp("sendhdr %d bytes, sent %d res %d\n", size, buf->sent, res);
1370         if (res >= 0) {
1371                 conn->txdata_octets += res;
1372                 buf->sent += res;
1373                 if (size != res)
1374                         return -EAGAIN;
1375                 return 0;
1376         } else if (res == -EAGAIN) {
1377                 conn->sendpage_failures_cnt++;
1378                 set_bit(SUSPEND_BIT, &conn->suspend_tx);
1379         } else if (res == -EPIPE)
1380                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1381
1382         return res;
1383 }
1384
1385 /**
1386  * iscsi_sendpage - send one page of iSCSI Data-Out.
1387  * @conn: iscsi connection
1388  * @buf: buffer to write from
1389  * @count: remaining data
1390  * @sent: number of bytes sent
1391  *
1392  * Notes:
1393  *      (Tx, Fast Path)
1394  **/
1395 static inline int
1396 iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf,
1397                int *count, int *sent)
1398 {
1399         struct socket *sk = conn->sock;
1400         int flags = 0; /* MSG_DONTWAIT; */
1401         int res, size;
1402
1403         size = buf->sg.length - buf->sent;
1404         BUG_ON(buf->sent + size > buf->sg.length);
1405         if (size > *count)
1406                 size = *count;
1407         if (buf->sent + size != buf->sg.length || *count != size)
1408                 flags |= MSG_MORE;
1409
1410         res = iscsi_send(sk, buf, size, flags);
1411         debug_tcp("sendpage: %d bytes, sent %d left %d sent %d res %d\n",
1412                   size, buf->sent, *count, *sent, res);
1413         if (res >= 0) {
1414                 conn->txdata_octets += res;
1415                 buf->sent += res;
1416                 *count -= res;
1417                 *sent += res;
1418                 if (size != res)
1419                         return -EAGAIN;
1420                 return 0;
1421         } else if (res == -EAGAIN) {
1422                 conn->sendpage_failures_cnt++;
1423                 set_bit(SUSPEND_BIT, &conn->suspend_tx);
1424         } else if (res == -EPIPE)
1425                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1426
1427         return res;
1428 }
1429
1430 static inline void
1431 iscsi_data_digest_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1432 {
1433         BUG_ON(!conn->data_tx_tfm);
1434         crypto_digest_init(conn->data_tx_tfm);
1435         ctask->digest_count = 4;
1436 }
1437
1438 static inline void
1439 iscsi_buf_data_digest_update(struct iscsi_conn *conn, struct iscsi_buf *buf)
1440 {
1441         struct scatterlist sg;
1442
1443         if (buf->sg.offset != -1)
1444                 crypto_digest_update(conn->data_tx_tfm, &buf->sg, 1);
1445         else {
1446                 sg_init_one(&sg, (char *)buf->sg.page, buf->sg.length);
1447                 crypto_digest_update(conn->data_tx_tfm, &sg, 1);
1448         }
1449 }
1450
1451 static inline int
1452 iscsi_digest_final_send(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1453                         struct iscsi_buf *buf, uint32_t *digest, int final)
1454 {
1455         int rc = 0;
1456         int sent = 0;
1457
1458         if (final)
1459                 crypto_digest_final(conn->data_tx_tfm, (u8*)digest);
1460
1461         iscsi_buf_init_virt(buf, (char*)digest, 4);
1462         rc = iscsi_sendpage(conn, buf, &ctask->digest_count, &sent);
1463         if (rc) {
1464                 ctask->datadigest = *digest;
1465                 ctask->xmstate |= XMSTATE_DATA_DIGEST;
1466         } else
1467                 ctask->digest_count = 4;
1468         return rc;
1469 }
1470
1471 /**
1472  * iscsi_solicit_data_cont - initialize next Data-Out
1473  * @conn: iscsi connection
1474  * @ctask: scsi command task
1475  * @r2t: R2T info
1476  * @left: bytes left to transfer
1477  *
1478  * Notes:
1479  *      Initialize next Data-Out within this R2T sequence and continue
1480  *      to process next Scatter-Gather element(if any) of this SCSI command.
1481  *
1482  *      Called under connection lock.
1483  **/
1484 static void
1485 iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1486                         struct iscsi_r2t_info *r2t, int left)
1487 {
1488         struct iscsi_data *hdr;
1489         struct iscsi_data_task *dtask;
1490         struct scsi_cmnd *sc = ctask->sc;
1491         int new_offset;
1492
1493         dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
1494         BUG_ON(!dtask);
1495         hdr = &dtask->hdr;
1496         memset(hdr, 0, sizeof(struct iscsi_data));
1497         hdr->ttt = r2t->ttt;
1498         hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
1499         r2t->solicit_datasn++;
1500         hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1501         memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
1502         hdr->itt = ctask->hdr.itt;
1503         hdr->exp_statsn = r2t->exp_statsn;
1504         new_offset = r2t->data_offset + r2t->sent;
1505         hdr->offset = cpu_to_be32(new_offset);
1506         if (left > conn->max_xmit_dlength) {
1507                 hton24(hdr->dlength, conn->max_xmit_dlength);
1508                 r2t->data_count = conn->max_xmit_dlength;
1509         } else {
1510                 hton24(hdr->dlength, left);
1511                 r2t->data_count = left;
1512                 hdr->flags = ISCSI_FLAG_CMD_FINAL;
1513         }
1514         conn->dataout_pdus_cnt++;
1515
1516         iscsi_buf_init_virt(&r2t->headbuf, (char*)hdr,
1517                            sizeof(struct iscsi_hdr));
1518
1519         r2t->dtask = dtask;
1520
1521         if (sc->use_sg && !iscsi_buf_left(&r2t->sendbuf)) {
1522                 BUG_ON(ctask->bad_sg == r2t->sg);
1523                 iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1524                 r2t->sg += 1;
1525         } else
1526                 iscsi_buf_init_iov(&ctask->sendbuf,
1527                             (char*)sc->request_buffer + new_offset,
1528                             r2t->data_count);
1529
1530         list_add(&dtask->item, &ctask->dataqueue);
1531 }
1532
1533 static void
1534 iscsi_unsolicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1535 {
1536         struct iscsi_data *hdr;
1537         struct iscsi_data_task *dtask;
1538
1539         dtask = mempool_alloc(ctask->datapool, GFP_ATOMIC);
1540         BUG_ON(!dtask);
1541         hdr = &dtask->hdr;
1542         memset(hdr, 0, sizeof(struct iscsi_data));
1543         hdr->ttt = cpu_to_be32(ISCSI_RESERVED_TAG);
1544         hdr->datasn = cpu_to_be32(ctask->unsol_datasn);
1545         ctask->unsol_datasn++;
1546         hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1547         memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
1548         hdr->itt = ctask->hdr.itt;
1549         hdr->exp_statsn = cpu_to_be32(conn->exp_statsn);
1550         hdr->offset = cpu_to_be32(ctask->total_length -
1551                                   ctask->r2t_data_count -
1552                                   ctask->unsol_count);
1553         if (ctask->unsol_count > conn->max_xmit_dlength) {
1554                 hton24(hdr->dlength, conn->max_xmit_dlength);
1555                 ctask->data_count = conn->max_xmit_dlength;
1556                 hdr->flags = 0;
1557         } else {
1558                 hton24(hdr->dlength, ctask->unsol_count);
1559                 ctask->data_count = ctask->unsol_count;
1560                 hdr->flags = ISCSI_FLAG_CMD_FINAL;
1561         }
1562
1563         iscsi_buf_init_virt(&ctask->headbuf, (char*)hdr,
1564                            sizeof(struct iscsi_hdr));
1565
1566         list_add(&dtask->item, &ctask->dataqueue);
1567
1568         ctask->dtask = dtask;
1569 }
1570
1571 /**
1572  * iscsi_cmd_init - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
1573  * @conn: iscsi connection
1574  * @ctask: scsi command task
1575  * @sc: scsi command
1576  **/
1577 static void
1578 iscsi_cmd_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
1579                 struct scsi_cmnd *sc)
1580 {
1581         struct iscsi_session *session = conn->session;
1582
1583         BUG_ON(__kfifo_len(ctask->r2tqueue));
1584
1585         ctask->sc = sc;
1586         ctask->conn = conn;
1587         ctask->hdr.opcode = ISCSI_OP_SCSI_CMD;
1588         ctask->hdr.flags = ISCSI_ATTR_SIMPLE;
1589         int_to_scsilun(sc->device->lun, (struct scsi_lun *)ctask->hdr.lun);
1590         ctask->hdr.itt = ctask->itt | (conn->id << CID_SHIFT) |
1591                          (session->age << AGE_SHIFT);
1592         ctask->hdr.data_length = cpu_to_be32(sc->request_bufflen);
1593         ctask->hdr.cmdsn = cpu_to_be32(session->cmdsn); session->cmdsn++;
1594         ctask->hdr.exp_statsn = cpu_to_be32(conn->exp_statsn);
1595         memcpy(ctask->hdr.cdb, sc->cmnd, sc->cmd_len);
1596         memset(&ctask->hdr.cdb[sc->cmd_len], 0, MAX_COMMAND_SIZE - sc->cmd_len);
1597
1598         ctask->mtask = NULL;
1599         ctask->sent = 0;
1600         ctask->sg_count = 0;
1601
1602         ctask->total_length = sc->request_bufflen;
1603
1604         if (sc->sc_data_direction == DMA_TO_DEVICE) {
1605                 ctask->exp_r2tsn = 0;
1606                 ctask->hdr.flags |= ISCSI_FLAG_CMD_WRITE;
1607                 BUG_ON(ctask->total_length == 0);
1608                 if (sc->use_sg) {
1609                         struct scatterlist *sg = sc->request_buffer;
1610
1611                         iscsi_buf_init_sg(&ctask->sendbuf,
1612                                           &sg[ctask->sg_count++]);
1613                         ctask->sg = sg;
1614                         ctask->bad_sg = sg + sc->use_sg;
1615                 } else {
1616                         iscsi_buf_init_iov(&ctask->sendbuf, sc->request_buffer,
1617                                         sc->request_bufflen);
1618                 }
1619
1620                 /*
1621                  * Write counters:
1622                  *
1623                  *      imm_count       bytes to be sent right after
1624                  *                      SCSI PDU Header
1625                  *
1626                  *      unsol_count     bytes(as Data-Out) to be sent
1627                  *                      without R2T ack right after
1628                  *                      immediate data
1629                  *
1630                  *      r2t_data_count  bytes to be sent via R2T ack's
1631                  *
1632                  *      pad_count       bytes to be sent as zero-padding
1633                  */
1634                 ctask->imm_count = 0;
1635                 ctask->unsol_count = 0;
1636                 ctask->unsol_datasn = 0;
1637                 ctask->xmstate = XMSTATE_W_HDR;
1638                 /* calculate write padding */
1639                 ctask->pad_count = ctask->total_length & (ISCSI_PAD_LEN-1);
1640                 if (ctask->pad_count) {
1641                         ctask->pad_count = ISCSI_PAD_LEN - ctask->pad_count;
1642                         debug_scsi("write padding %d bytes\n",
1643                                 ctask->pad_count);
1644                         ctask->xmstate |= XMSTATE_W_PAD;
1645                 }
1646                 if (session->imm_data_en) {
1647                         if (ctask->total_length >= session->first_burst)
1648                                 ctask->imm_count = min(session->first_burst,
1649                                                         conn->max_xmit_dlength);
1650                         else
1651                                 ctask->imm_count = min(ctask->total_length,
1652                                                         conn->max_xmit_dlength);
1653                         hton24(ctask->hdr.dlength, ctask->imm_count);
1654                         ctask->xmstate |= XMSTATE_IMM_DATA;
1655                 } else
1656                         zero_data(ctask->hdr.dlength);
1657
1658                 if (!session->initial_r2t_en)
1659                         ctask->unsol_count = min(session->first_burst,
1660                                 ctask->total_length) - ctask->imm_count;
1661                 if (!ctask->unsol_count)
1662                         /* No unsolicit Data-Out's */
1663                         ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL;
1664                 else
1665                         ctask->xmstate |= XMSTATE_UNS_HDR | XMSTATE_UNS_INIT;
1666
1667                 ctask->r2t_data_count = ctask->total_length -
1668                                     ctask->imm_count -
1669                                     ctask->unsol_count;
1670
1671                 debug_scsi("cmd [itt %x total %d imm %d imm_data %d "
1672                            "r2t_data %d]\n",
1673                            ctask->itt, ctask->total_length, ctask->imm_count,
1674                            ctask->unsol_count, ctask->r2t_data_count);
1675         } else {
1676                 ctask->hdr.flags |= ISCSI_FLAG_CMD_FINAL;
1677                 if (sc->sc_data_direction == DMA_FROM_DEVICE)
1678                         ctask->hdr.flags |= ISCSI_FLAG_CMD_READ;
1679                 ctask->datasn = 0;
1680                 ctask->xmstate = XMSTATE_R_HDR;
1681                 zero_data(ctask->hdr.dlength);
1682         }
1683
1684         iscsi_buf_init_virt(&ctask->headbuf, (char*)&ctask->hdr,
1685                             sizeof(struct iscsi_hdr));
1686         conn->scsicmd_pdus_cnt++;
1687 }
1688
1689 /**
1690  * iscsi_mtask_xmit - xmit management(immediate) task
1691  * @conn: iscsi connection
1692  * @mtask: task management task
1693  *
1694  * Notes:
1695  *      The function can return -EAGAIN in which case caller must
1696  *      call it again later, or recover. '0' return code means successful
1697  *      xmit.
1698  *
1699  *      Management xmit state machine consists of two states:
1700  *              IN_PROGRESS_IMM_HEAD - PDU Header xmit in progress
1701  *              IN_PROGRESS_IMM_DATA - PDU Data xmit in progress
1702  **/
1703 static int
1704 iscsi_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask)
1705 {
1706
1707         debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n",
1708                 conn->id, mtask->xmstate, mtask->itt);
1709
1710         if (mtask->xmstate & XMSTATE_IMM_HDR) {
1711                 mtask->xmstate &= ~XMSTATE_IMM_HDR;
1712                 if (mtask->data_count)
1713                         mtask->xmstate |= XMSTATE_IMM_DATA;
1714                 if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE &&
1715                     conn->stop_stage != STOP_CONN_RECOVER &&
1716                     conn->hdrdgst_en)
1717                         iscsi_hdr_digest(conn, &mtask->headbuf,
1718                                         (u8*)mtask->hdrext);
1719                 if (iscsi_sendhdr(conn, &mtask->headbuf, mtask->data_count)) {
1720                         mtask->xmstate |= XMSTATE_IMM_HDR;
1721                         if (mtask->data_count)
1722                                 mtask->xmstate &= ~XMSTATE_IMM_DATA;
1723                         return -EAGAIN;
1724                 }
1725         }
1726
1727         if (mtask->xmstate & XMSTATE_IMM_DATA) {
1728                 BUG_ON(!mtask->data_count);
1729                 mtask->xmstate &= ~XMSTATE_IMM_DATA;
1730                 /* FIXME: implement.
1731                  * Virtual buffer could be spreaded across multiple pages...
1732                  */
1733                 do {
1734                         if (iscsi_sendpage(conn, &mtask->sendbuf,
1735                                    &mtask->data_count, &mtask->sent)) {
1736                                 mtask->xmstate |= XMSTATE_IMM_DATA;
1737                                 return -EAGAIN;
1738                         }
1739                 } while (mtask->data_count);
1740         }
1741
1742         BUG_ON(mtask->xmstate != XMSTATE_IDLE);
1743         return 0;
1744 }
1745
1746 static inline int
1747 handle_xmstate_r_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1748 {
1749         ctask->xmstate &= ~XMSTATE_R_HDR;
1750         if (conn->hdrdgst_en)
1751                 iscsi_hdr_digest(conn, &ctask->headbuf, (u8*)ctask->hdrext);
1752         if (!iscsi_sendhdr(conn, &ctask->headbuf, 0)) {
1753                 BUG_ON(ctask->xmstate != XMSTATE_IDLE);
1754                 return 0; /* wait for Data-In */
1755         }
1756         ctask->xmstate |= XMSTATE_R_HDR;
1757         return -EAGAIN;
1758 }
1759
1760 static inline int
1761 handle_xmstate_w_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1762 {
1763         ctask->xmstate &= ~XMSTATE_W_HDR;
1764         if (conn->hdrdgst_en)
1765                 iscsi_hdr_digest(conn, &ctask->headbuf, (u8*)ctask->hdrext);
1766         if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->imm_count)) {
1767                 ctask->xmstate |= XMSTATE_W_HDR;
1768                 return -EAGAIN;
1769         }
1770         return 0;
1771 }
1772
1773 static inline int
1774 handle_xmstate_data_digest(struct iscsi_conn *conn,
1775                            struct iscsi_cmd_task *ctask)
1776 {
1777         ctask->xmstate &= ~XMSTATE_DATA_DIGEST;
1778         debug_tcp("resent data digest 0x%x\n", ctask->datadigest);
1779         if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
1780                                     &ctask->datadigest, 0)) {
1781                 ctask->xmstate |= XMSTATE_DATA_DIGEST;
1782                 debug_tcp("resent data digest 0x%x fail!\n",
1783                           ctask->datadigest);
1784                 return -EAGAIN;
1785         }
1786         return 0;
1787 }
1788
1789 static inline int
1790 handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1791 {
1792         BUG_ON(!ctask->imm_count);
1793         ctask->xmstate &= ~XMSTATE_IMM_DATA;
1794
1795         if (conn->datadgst_en) {
1796                 iscsi_data_digest_init(conn, ctask);
1797                 ctask->immdigest = 0;
1798         }
1799
1800         for (;;) {
1801                 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->imm_count,
1802                                    &ctask->sent)) {
1803                         ctask->xmstate |= XMSTATE_IMM_DATA;
1804                         if (conn->datadgst_en) {
1805                                 crypto_digest_final(conn->data_tx_tfm,
1806                                                 (u8*)&ctask->immdigest);
1807                                 debug_tcp("tx imm sendpage fail 0x%x\n",
1808                                           ctask->datadigest);
1809                         }
1810                         return -EAGAIN;
1811                 }
1812                 if (conn->datadgst_en)
1813                         iscsi_buf_data_digest_update(conn, &ctask->sendbuf);
1814
1815                 if (!ctask->imm_count)
1816                         break;
1817                 iscsi_buf_init_sg(&ctask->sendbuf,
1818                                   &ctask->sg[ctask->sg_count++]);
1819         }
1820
1821         if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) {
1822                 if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
1823                                             &ctask->immdigest, 1)) {
1824                         debug_tcp("sending imm digest 0x%x fail!\n",
1825                                   ctask->immdigest);
1826                         return -EAGAIN;
1827                 }
1828                 debug_tcp("sending imm digest 0x%x\n", ctask->immdigest);
1829         }
1830
1831         return 0;
1832 }
1833
1834 static inline int
1835 handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1836 {
1837         struct iscsi_data_task *dtask;
1838
1839         ctask->xmstate |= XMSTATE_UNS_DATA;
1840         if (ctask->xmstate & XMSTATE_UNS_INIT) {
1841                 iscsi_unsolicit_data_init(conn, ctask);
1842                 BUG_ON(!ctask->dtask);
1843                 dtask = ctask->dtask;
1844                 if (conn->hdrdgst_en)
1845                         iscsi_hdr_digest(conn, &ctask->headbuf,
1846                                         (u8*)dtask->hdrext);
1847                 ctask->xmstate &= ~XMSTATE_UNS_INIT;
1848         }
1849         if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->data_count)) {
1850                 ctask->xmstate &= ~XMSTATE_UNS_DATA;
1851                 ctask->xmstate |= XMSTATE_UNS_HDR;
1852                 return -EAGAIN;
1853         }
1854
1855         debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n",
1856                    ctask->itt, ctask->unsol_count, ctask->sent);
1857         return 0;
1858 }
1859
1860 static inline int
1861 handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1862 {
1863         struct iscsi_data_task *dtask = ctask->dtask;
1864
1865         BUG_ON(!ctask->data_count);
1866         ctask->xmstate &= ~XMSTATE_UNS_DATA;
1867
1868         if (conn->datadgst_en) {
1869                 iscsi_data_digest_init(conn, ctask);
1870                 dtask->digest = 0;
1871         }
1872
1873         for (;;) {
1874                 int start = ctask->sent;
1875
1876                 if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->data_count,
1877                                    &ctask->sent)) {
1878                         ctask->unsol_count -= ctask->sent - start;
1879                         ctask->xmstate |= XMSTATE_UNS_DATA;
1880                         /* will continue with this ctask later.. */
1881                         if (conn->datadgst_en) {
1882                                 crypto_digest_final(conn->data_tx_tfm,
1883                                                 (u8 *)&dtask->digest);
1884                                 debug_tcp("tx uns data fail 0x%x\n",
1885                                           dtask->digest);
1886                         }
1887                         return -EAGAIN;
1888                 }
1889
1890                 BUG_ON(ctask->sent > ctask->total_length);
1891                 ctask->unsol_count -= ctask->sent - start;
1892
1893                 /*
1894                  * XXX:we may run here with un-initial sendbuf.
1895                  * so pass it
1896                  */
1897                 if (conn->datadgst_en && ctask->sent - start > 0)
1898                         iscsi_buf_data_digest_update(conn, &ctask->sendbuf);
1899
1900                 if (!ctask->data_count)
1901                         break;
1902                 iscsi_buf_init_sg(&ctask->sendbuf,
1903                                   &ctask->sg[ctask->sg_count++]);
1904         }
1905         BUG_ON(ctask->unsol_count < 0);
1906
1907         /*
1908          * Done with the Data-Out. Next, check if we need
1909          * to send another unsolicited Data-Out.
1910          */
1911         if (ctask->unsol_count) {
1912                 if (conn->datadgst_en) {
1913                         if (iscsi_digest_final_send(conn, ctask,
1914                                                     &dtask->digestbuf,
1915                                                     &dtask->digest, 1)) {
1916                                 debug_tcp("send uns digest 0x%x fail\n",
1917                                           dtask->digest);
1918                                 return -EAGAIN;
1919                         }
1920                         debug_tcp("sending uns digest 0x%x, more uns\n",
1921                                   dtask->digest);
1922                 }
1923                 ctask->xmstate |= XMSTATE_UNS_INIT;
1924                 return 1;
1925         }
1926
1927         if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) {
1928                 if (iscsi_digest_final_send(conn, ctask,
1929                                             &dtask->digestbuf,
1930                                             &dtask->digest, 1)) {
1931                         debug_tcp("send last uns digest 0x%x fail\n",
1932                                    dtask->digest);
1933                         return -EAGAIN;
1934                 }
1935                 debug_tcp("sending uns digest 0x%x\n",dtask->digest);
1936         }
1937
1938         return 0;
1939 }
1940
1941 static inline int
1942 handle_xmstate_sol_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
1943 {
1944         struct iscsi_session *session = conn->session;
1945         struct iscsi_r2t_info *r2t = ctask->r2t;
1946         struct iscsi_data_task *dtask = r2t->dtask;
1947         int left;
1948
1949         ctask->xmstate &= ~XMSTATE_SOL_DATA;
1950         ctask->dtask = dtask;
1951
1952         if (conn->datadgst_en) {
1953                 iscsi_data_digest_init(conn, ctask);
1954                 dtask->digest = 0;
1955         }
1956 solicit_again:
1957         /*
1958          * send Data-Out whitnin this R2T sequence.
1959          */
1960         if (!r2t->data_count)
1961                 goto data_out_done;
1962
1963         if (iscsi_sendpage(conn, &r2t->sendbuf, &r2t->data_count, &r2t->sent)) {
1964                 ctask->xmstate |= XMSTATE_SOL_DATA;
1965                 /* will continue with this ctask later.. */
1966                 if (conn->datadgst_en) {
1967                         crypto_digest_final(conn->data_tx_tfm,
1968                                           (u8 *)&dtask->digest);
1969                         debug_tcp("r2t data send fail 0x%x\n", dtask->digest);
1970                 }
1971                 return -EAGAIN;
1972         }
1973
1974         BUG_ON(r2t->data_count < 0);
1975         if (conn->datadgst_en)
1976                 iscsi_buf_data_digest_update(conn, &r2t->sendbuf);
1977
1978         if (r2t->data_count) {
1979                 BUG_ON(ctask->sc->use_sg == 0);
1980                 if (!iscsi_buf_left(&r2t->sendbuf)) {
1981                         BUG_ON(ctask->bad_sg == r2t->sg);
1982                         iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
1983                         r2t->sg += 1;
1984                 }
1985                 goto solicit_again;
1986         }
1987
1988 data_out_done:
1989         /*
1990          * Done with this Data-Out. Next, check if we have
1991          * to send another Data-Out for this R2T.
1992          */
1993         BUG_ON(r2t->data_length - r2t->sent < 0);
1994         left = r2t->data_length - r2t->sent;
1995         if (left) {
1996                 if (conn->datadgst_en) {
1997                         if (iscsi_digest_final_send(conn, ctask,
1998                                                     &dtask->digestbuf,
1999                                                     &dtask->digest, 1)) {
2000                                 debug_tcp("send r2t data digest 0x%x"
2001                                           "fail\n", dtask->digest);
2002                                 return -EAGAIN;
2003                         }
2004                         debug_tcp("r2t data send digest 0x%x\n",
2005                                   dtask->digest);
2006                 }
2007                 iscsi_solicit_data_cont(conn, ctask, r2t, left);
2008                 ctask->xmstate |= XMSTATE_SOL_DATA;
2009                 ctask->xmstate &= ~XMSTATE_SOL_HDR;
2010                 return 1;
2011         }
2012
2013         /*
2014          * Done with this R2T. Check if there are more
2015          * outstanding R2Ts ready to be processed.
2016          */
2017         BUG_ON(ctask->r2t_data_count - r2t->data_length < 0);
2018         if (conn->datadgst_en) {
2019                 if (iscsi_digest_final_send(conn, ctask, &dtask->digestbuf,
2020                                             &dtask->digest, 1)) {
2021                         debug_tcp("send last r2t data digest 0x%x"
2022                                   "fail\n", dtask->digest);
2023                         return -EAGAIN;
2024                 }
2025                 debug_tcp("r2t done dout digest 0x%x\n", dtask->digest);
2026         }
2027
2028         ctask->r2t_data_count -= r2t->data_length;
2029         ctask->r2t = NULL;
2030         spin_lock_bh(&session->lock);
2031         __kfifo_put(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
2032         spin_unlock_bh(&session->lock);
2033         if (__kfifo_get(ctask->r2tqueue, (void*)&r2t, sizeof(void*))) {
2034                 ctask->r2t = r2t;
2035                 ctask->xmstate |= XMSTATE_SOL_DATA;
2036                 ctask->xmstate &= ~XMSTATE_SOL_HDR;
2037                 return 1;
2038         }
2039
2040         return 0;
2041 }
2042
2043 static inline int
2044 handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2045 {
2046         struct iscsi_data_task *dtask = ctask->dtask;
2047         int sent;
2048
2049         ctask->xmstate &= ~XMSTATE_W_PAD;
2050         iscsi_buf_init_virt(&ctask->sendbuf, (char*)&ctask->pad,
2051                             ctask->pad_count);
2052         if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->pad_count, &sent)) {
2053                 ctask->xmstate |= XMSTATE_W_PAD;
2054                 return -EAGAIN;
2055         }
2056
2057         if (conn->datadgst_en) {
2058                 iscsi_buf_data_digest_update(conn, &ctask->sendbuf);
2059                 /* imm data? */
2060                 if (!dtask) {
2061                         if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
2062                                                     &ctask->immdigest, 1)) {
2063                                 debug_tcp("send padding digest 0x%x"
2064                                           "fail!\n", ctask->immdigest);
2065                                 return -EAGAIN;
2066                         }
2067                         debug_tcp("done with padding, digest 0x%x\n",
2068                                   ctask->datadigest);
2069                 } else {
2070                         if (iscsi_digest_final_send(conn, ctask,
2071                                                     &dtask->digestbuf,
2072                                                     &dtask->digest, 1)) {
2073                                 debug_tcp("send padding digest 0x%x"
2074                                           "fail\n", dtask->digest);
2075                                 return -EAGAIN;
2076                         }
2077                         debug_tcp("done with padding, digest 0x%x\n",
2078                                   dtask->digest);
2079                 }
2080         }
2081
2082         return 0;
2083 }
2084
2085 static int
2086 iscsi_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
2087 {
2088         int rc = 0;
2089
2090         debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n",
2091                 conn->id, ctask->xmstate, ctask->itt);
2092
2093         /*
2094          * serialize with TMF AbortTask
2095          */
2096         if (ctask->mtask)
2097                 return rc;
2098
2099         if (ctask->xmstate & XMSTATE_R_HDR) {
2100                 rc = handle_xmstate_r_hdr(conn, ctask);
2101                 return rc;
2102         }
2103
2104         if (ctask->xmstate & XMSTATE_W_HDR) {
2105                 rc = handle_xmstate_w_hdr(conn, ctask);
2106                 if (rc)
2107                         return rc;
2108         }
2109
2110         /* XXX: for data digest xmit recover */
2111         if (ctask->xmstate & XMSTATE_DATA_DIGEST) {
2112                 rc = handle_xmstate_data_digest(conn, ctask);
2113                 if (rc)
2114                         return rc;
2115         }
2116
2117         if (ctask->xmstate & XMSTATE_IMM_DATA) {
2118                 rc = handle_xmstate_imm_data(conn, ctask);
2119                 if (rc)
2120                         return rc;
2121         }
2122
2123         if (ctask->xmstate & XMSTATE_UNS_HDR) {
2124                 BUG_ON(!ctask->unsol_count);
2125                 ctask->xmstate &= ~XMSTATE_UNS_HDR;
2126 unsolicit_head_again:
2127                 rc = handle_xmstate_uns_hdr(conn, ctask);
2128                 if (rc)
2129                         return rc;
2130         }
2131
2132         if (ctask->xmstate & XMSTATE_UNS_DATA) {
2133                 rc = handle_xmstate_uns_data(conn, ctask);
2134                 if (rc == 1)
2135                         goto unsolicit_head_again;
2136                 else if (rc)
2137                         return rc;
2138                 goto done;
2139         }
2140
2141         if (ctask->xmstate & XMSTATE_SOL_HDR) {
2142                 struct iscsi_r2t_info *r2t;
2143
2144                 ctask->xmstate &= ~XMSTATE_SOL_HDR;
2145                 ctask->xmstate |= XMSTATE_SOL_DATA;
2146                 if (!ctask->r2t)
2147                         __kfifo_get(ctask->r2tqueue, (void*)&ctask->r2t,
2148                                     sizeof(void*));
2149 solicit_head_again:
2150                 r2t = ctask->r2t;
2151                 if (conn->hdrdgst_en)
2152                         iscsi_hdr_digest(conn, &r2t->headbuf,
2153                                         (u8*)r2t->dtask->hdrext);
2154                 if (iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count)) {
2155                         ctask->xmstate &= ~XMSTATE_SOL_DATA;
2156                         ctask->xmstate |= XMSTATE_SOL_HDR;
2157                         return -EAGAIN;
2158                 }
2159
2160                 debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n",
2161                         r2t->solicit_datasn - 1, ctask->itt, r2t->data_count,
2162                         r2t->sent);
2163         }
2164
2165         if (ctask->xmstate & XMSTATE_SOL_DATA) {
2166                 rc = handle_xmstate_sol_data(conn, ctask);
2167                 if (rc == 1)
2168                         goto solicit_head_again;
2169                 if (rc)
2170                         return rc;
2171         }
2172
2173 done:
2174         /*
2175          * Last thing to check is whether we need to send write
2176          * padding. Note that we check for xmstate equality, not just the bit.
2177          */
2178         if (ctask->xmstate == XMSTATE_W_PAD)
2179                 rc = handle_xmstate_w_pad(conn, ctask);
2180
2181         return rc;
2182 }
2183
2184 /**
2185  * iscsi_data_xmit - xmit any command into the scheduled connection
2186  * @conn: iscsi connection
2187  *
2188  * Notes:
2189  *      The function can return -EAGAIN in which case the caller must
2190  *      re-schedule it again later or recover. '0' return code means
2191  *      successful xmit.
2192  **/
2193 static int
2194 iscsi_data_xmit(struct iscsi_conn *conn)
2195 {
2196         if (unlikely(conn->suspend_tx)) {
2197                 debug_tcp("conn %d Tx suspended!\n", conn->id);
2198                 return 0;
2199         }
2200
2201         /*
2202          * Transmit in the following order:
2203          *
2204          * 1) un-finished xmit (ctask or mtask)
2205          * 2) immediate control PDUs
2206          * 3) write data
2207          * 4) SCSI commands
2208          * 5) non-immediate control PDUs
2209          *
2210          * No need to lock around __kfifo_get as long as
2211          * there's one producer and one consumer.
2212          */
2213
2214         BUG_ON(conn->ctask && conn->mtask);
2215
2216         if (conn->ctask) {
2217                 if (iscsi_ctask_xmit(conn, conn->ctask))
2218                         goto again;
2219                 /* done with this in-progress ctask */
2220                 conn->ctask = NULL;
2221         }
2222         if (conn->mtask) {
2223                 if (iscsi_mtask_xmit(conn, conn->mtask))
2224                         goto again;
2225                 /* done with this in-progress mtask */
2226                 conn->mtask = NULL;
2227         }
2228
2229         /* process immediate first */
2230         if (unlikely(__kfifo_len(conn->immqueue))) {
2231                 struct iscsi_session *session = conn->session;
2232                 while (__kfifo_get(conn->immqueue, (void*)&conn->mtask,
2233                                    sizeof(void*))) {
2234                         if (iscsi_mtask_xmit(conn, conn->mtask))
2235                                 goto again;
2236
2237                         if (conn->mtask->hdr.itt ==
2238                                         cpu_to_be32(ISCSI_RESERVED_TAG)) {
2239                                 spin_lock_bh(&session->lock);
2240                                 __kfifo_put(session->mgmtpool.queue,
2241                                             (void*)&conn->mtask, sizeof(void*));
2242                                 spin_unlock_bh(&session->lock);
2243                         }
2244                 }
2245                 /* done with this mtask */
2246                 conn->mtask = NULL;
2247         }
2248
2249         /* process write queue */
2250         while (__kfifo_get(conn->writequeue, (void*)&conn->ctask,
2251                            sizeof(void*))) {
2252                 if (iscsi_ctask_xmit(conn, conn->ctask))
2253                         goto again;
2254         }
2255
2256         /* process command queue */
2257         while (__kfifo_get(conn->xmitqueue, (void*)&conn->ctask,
2258                            sizeof(void*))) {
2259                 if (iscsi_ctask_xmit(conn, conn->ctask))
2260                         goto again;
2261         }
2262         /* done with this ctask */
2263         conn->ctask = NULL;
2264
2265         /* process the rest control plane PDUs, if any */
2266         if (unlikely(__kfifo_len(conn->mgmtqueue))) {
2267                 struct iscsi_session *session = conn->session;
2268
2269                 while (__kfifo_get(conn->mgmtqueue, (void*)&conn->mtask,
2270                                    sizeof(void*))) {
2271                         if (iscsi_mtask_xmit(conn, conn->mtask))
2272                                 goto again;
2273
2274                         if (conn->mtask->hdr.itt ==
2275                                         cpu_to_be32(ISCSI_RESERVED_TAG)) {
2276                                 spin_lock_bh(&session->lock);
2277                                 __kfifo_put(session->mgmtpool.queue,
2278                                             (void*)&conn->mtask,
2279                                             sizeof(void*));
2280                                 spin_unlock_bh(&session->lock);
2281                         }
2282                 }
2283                 /* done with this mtask */
2284                 conn->mtask = NULL;
2285         }
2286
2287         return 0;
2288
2289 again:
2290         if (unlikely(conn->suspend_tx))
2291                 return 0;
2292
2293         return -EAGAIN;
2294 }
2295
2296 static void
2297 iscsi_xmitworker(void *data)
2298 {
2299         struct iscsi_conn *conn = data;
2300
2301         /*
2302          * serialize Xmit worker on a per-connection basis.
2303          */
2304         mutex_lock(&conn->xmitmutex);
2305         if (iscsi_data_xmit(conn))
2306                 schedule_work(&conn->xmitwork);
2307         mutex_unlock(&conn->xmitmutex);
2308 }
2309
2310 #define FAILURE_BAD_HOST                1
2311 #define FAILURE_SESSION_FAILED          2
2312 #define FAILURE_SESSION_FREED           3
2313 #define FAILURE_WINDOW_CLOSED           4
2314 #define FAILURE_SESSION_TERMINATE       5
2315
2316 static int
2317 iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *))
2318 {
2319         struct Scsi_Host *host;
2320         int reason = 0;
2321         struct iscsi_session *session;
2322         struct iscsi_conn *conn = NULL;
2323         struct iscsi_cmd_task *ctask = NULL;
2324
2325         sc->scsi_done = done;
2326         sc->result = 0;
2327
2328         host = sc->device->host;
2329         session = iscsi_hostdata(host->hostdata);
2330         BUG_ON(host != session->host);
2331
2332         spin_lock(&session->lock);
2333
2334         if (session->state != ISCSI_STATE_LOGGED_IN) {
2335                 if (session->state == ISCSI_STATE_FAILED) {
2336                         reason = FAILURE_SESSION_FAILED;
2337                         goto reject;
2338                 } else if (session->state == ISCSI_STATE_TERMINATE) {
2339                         reason = FAILURE_SESSION_TERMINATE;
2340                         goto fault;
2341                 }
2342                 reason = FAILURE_SESSION_FREED;
2343                 goto fault;
2344         }
2345
2346         /*
2347          * Check for iSCSI window and take care of CmdSN wrap-around
2348          */
2349         if ((int)(session->max_cmdsn - session->cmdsn) < 0) {
2350                 reason = FAILURE_WINDOW_CLOSED;
2351                 goto reject;
2352         }
2353
2354         conn = session->leadconn;
2355
2356         __kfifo_get(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
2357         BUG_ON(ctask->sc);
2358
2359         sc->SCp.phase = session->age;
2360         sc->SCp.ptr = (char*)ctask;
2361         iscsi_cmd_init(conn, ctask, sc);
2362
2363         __kfifo_put(conn->xmitqueue, (void*)&ctask, sizeof(void*));
2364         debug_scsi(
2365                "ctask enq [%s cid %d sc %lx itt 0x%x len %d cmdsn %d win %d]\n",
2366                 sc->sc_data_direction == DMA_TO_DEVICE ? "write" : "read",
2367                 conn->id, (long)sc, ctask->itt, sc->request_bufflen,
2368                 session->cmdsn, session->max_cmdsn - session->exp_cmdsn + 1);
2369         spin_unlock(&session->lock);
2370
2371         if (!in_interrupt() && mutex_trylock(&conn->xmitmutex)) {
2372                 spin_unlock_irq(host->host_lock);
2373                 if (iscsi_data_xmit(conn))
2374                         schedule_work(&conn->xmitwork);
2375                 mutex_unlock(&conn->xmitmutex);
2376                 spin_lock_irq(host->host_lock);
2377         } else
2378                 schedule_work(&conn->xmitwork);
2379
2380         return 0;
2381
2382 reject:
2383         spin_unlock(&session->lock);
2384         debug_scsi("cmd 0x%x rejected (%d)\n", sc->cmnd[0], reason);
2385         return SCSI_MLQUEUE_HOST_BUSY;
2386
2387 fault:
2388         spin_unlock(&session->lock);
2389         printk(KERN_ERR "iscsi_tcp: cmd 0x%x is not queued (%d)\n",
2390                sc->cmnd[0], reason);
2391         sc->sense_buffer[0] = 0x70;
2392         sc->sense_buffer[2] = NOT_READY;
2393         sc->sense_buffer[7] = 0x6;
2394         sc->sense_buffer[12] = 0x08;
2395         sc->sense_buffer[13] = 0x00;
2396         sc->result = (DID_NO_CONNECT << 16);
2397         sc->resid = sc->request_bufflen;
2398         sc->scsi_done(sc);
2399         return 0;
2400 }
2401
2402 static int
2403 iscsi_change_queue_depth(struct scsi_device *sdev, int depth)
2404 {
2405         if (depth > ISCSI_MAX_CMD_PER_LUN)
2406                 depth = ISCSI_MAX_CMD_PER_LUN;
2407         scsi_adjust_queue_depth(sdev, scsi_get_tag_type(sdev), depth);
2408         return sdev->queue_depth;
2409 }
2410
2411 static int
2412 iscsi_pool_init(struct iscsi_queue *q, int max, void ***items, int item_size)
2413 {
2414         int i;
2415
2416         *items = kmalloc(max * sizeof(void*), GFP_KERNEL);
2417         if (*items == NULL)
2418                 return -ENOMEM;
2419
2420         q->max = max;
2421         q->pool = kmalloc(max * sizeof(void*), GFP_KERNEL);
2422         if (q->pool == NULL) {
2423                 kfree(*items);
2424                 return -ENOMEM;
2425         }
2426
2427         q->queue = kfifo_init((void*)q->pool, max * sizeof(void*),
2428                               GFP_KERNEL, NULL);
2429         if (q->queue == ERR_PTR(-ENOMEM)) {
2430                 kfree(q->pool);
2431                 kfree(*items);
2432                 return -ENOMEM;
2433         }
2434
2435         for (i = 0; i < max; i++) {
2436                 q->pool[i] = kmalloc(item_size, GFP_KERNEL);
2437                 if (q->pool[i] == NULL) {
2438                         int j;
2439
2440                         for (j = 0; j < i; j++)
2441                                 kfree(q->pool[j]);
2442
2443                         kfifo_free(q->queue);
2444                         kfree(q->pool);
2445                         kfree(*items);
2446                         return -ENOMEM;
2447                 }
2448                 memset(q->pool[i], 0, item_size);
2449                 (*items)[i] = q->pool[i];
2450                 __kfifo_put(q->queue, (void*)&q->pool[i], sizeof(void*));
2451         }
2452         return 0;
2453 }
2454
2455 static void
2456 iscsi_pool_free(struct iscsi_queue *q, void **items)
2457 {
2458         int i;
2459
2460         for (i = 0; i < q->max; i++)
2461                 kfree(items[i]);
2462         kfree(q->pool);
2463         kfree(items);
2464 }
2465
2466 static iscsi_connh_t
2467 iscsi_conn_create(iscsi_sessionh_t sessionh, uint32_t conn_idx)
2468 {
2469         struct iscsi_session *session = iscsi_ptr(sessionh);
2470         struct iscsi_conn *conn = NULL;
2471
2472         conn = kmalloc(sizeof(struct iscsi_conn), GFP_KERNEL);
2473         if (conn == NULL)
2474                 goto conn_alloc_fail;
2475         memset(conn, 0, sizeof(struct iscsi_conn));
2476
2477         conn->c_stage = ISCSI_CONN_INITIAL_STAGE;
2478         conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2479         conn->id = conn_idx;
2480         conn->exp_statsn = 0;
2481         conn->tmabort_state = TMABORT_INITIAL;
2482
2483         /* initial operational parameters */
2484         conn->hdr_size = sizeof(struct iscsi_hdr);
2485         conn->data_size = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
2486         conn->max_recv_dlength = DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH;
2487
2488         spin_lock_init(&conn->lock);
2489
2490         /* initialize general xmit PDU commands queue */
2491         conn->xmitqueue = kfifo_alloc(session->cmds_max * sizeof(void*),
2492                                         GFP_KERNEL, NULL);
2493         if (conn->xmitqueue == ERR_PTR(-ENOMEM))
2494                 goto xmitqueue_alloc_fail;
2495
2496         /* initialize write response PDU commands queue */
2497         conn->writequeue = kfifo_alloc(session->cmds_max * sizeof(void*),
2498                                         GFP_KERNEL, NULL);
2499         if (conn->writequeue == ERR_PTR(-ENOMEM))
2500                 goto writequeue_alloc_fail;
2501
2502         /* initialize general immediate & non-immediate PDU commands queue */
2503         conn->immqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*),
2504                                         GFP_KERNEL, NULL);
2505         if (conn->immqueue == ERR_PTR(-ENOMEM))
2506                 goto immqueue_alloc_fail;
2507
2508         conn->mgmtqueue = kfifo_alloc(session->mgmtpool_max * sizeof(void*),
2509                                         GFP_KERNEL, NULL);
2510         if (conn->mgmtqueue == ERR_PTR(-ENOMEM))
2511                 goto mgmtqueue_alloc_fail;
2512
2513         INIT_WORK(&conn->xmitwork, iscsi_xmitworker, conn);
2514
2515         /* allocate login_mtask used for the login/text sequences */
2516         spin_lock_bh(&session->lock);
2517         if (!__kfifo_get(session->mgmtpool.queue,
2518                          (void*)&conn->login_mtask,
2519                          sizeof(void*))) {
2520                 spin_unlock_bh(&session->lock);
2521                 goto login_mtask_alloc_fail;
2522         }
2523         spin_unlock_bh(&session->lock);
2524
2525         /* allocate initial PDU receive place holder */
2526         if (conn->data_size <= PAGE_SIZE)
2527                 conn->data = kmalloc(conn->data_size, GFP_KERNEL);
2528         else
2529                 conn->data = (void*)__get_free_pages(GFP_KERNEL,
2530                                         get_order(conn->data_size));
2531         if (!conn->data)
2532                 goto max_recv_dlenght_alloc_fail;
2533
2534         init_timer(&conn->tmabort_timer);
2535         mutex_init(&conn->xmitmutex);
2536         init_waitqueue_head(&conn->ehwait);
2537
2538         return iscsi_handle(conn);
2539
2540 max_recv_dlenght_alloc_fail:
2541         spin_lock_bh(&session->lock);
2542         __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask,
2543                     sizeof(void*));
2544         spin_unlock_bh(&session->lock);
2545 login_mtask_alloc_fail:
2546         kfifo_free(conn->mgmtqueue);
2547 mgmtqueue_alloc_fail:
2548         kfifo_free(conn->immqueue);
2549 immqueue_alloc_fail:
2550         kfifo_free(conn->writequeue);
2551 writequeue_alloc_fail:
2552         kfifo_free(conn->xmitqueue);
2553 xmitqueue_alloc_fail:
2554         kfree(conn);
2555 conn_alloc_fail:
2556         return iscsi_handle(NULL);
2557 }
2558
2559 static void
2560 iscsi_conn_destroy(iscsi_connh_t connh)
2561 {
2562         struct iscsi_conn *conn = iscsi_ptr(connh);
2563         struct iscsi_session *session = conn->session;
2564
2565         mutex_lock(&conn->xmitmutex);
2566         set_bit(SUSPEND_BIT, &conn->suspend_tx);
2567         if (conn->c_stage == ISCSI_CONN_INITIAL_STAGE && conn->sock) {
2568                 struct sock *sk = conn->sock->sk;
2569
2570                 /*
2571                  * conn_start() has never been called!
2572                  * need to cleanup the socket.
2573                  */
2574                 write_lock_bh(&sk->sk_callback_lock);
2575                 set_bit(SUSPEND_BIT, &conn->suspend_rx);
2576                 write_unlock_bh(&sk->sk_callback_lock);
2577
2578                 sock_hold(conn->sock->sk);
2579                 iscsi_conn_restore_callbacks(conn);
2580                 sock_put(conn->sock->sk);
2581                 sock_release(conn->sock);
2582                 conn->sock = NULL;
2583         }
2584
2585         spin_lock_bh(&session->lock);
2586         conn->c_stage = ISCSI_CONN_CLEANUP_WAIT;
2587         if (session->leadconn == conn) {
2588                 /*
2589                  * leading connection? then give up on recovery.
2590                  */
2591                 session->state = ISCSI_STATE_TERMINATE;
2592                 wake_up(&conn->ehwait);
2593         }
2594         spin_unlock_bh(&session->lock);
2595
2596         mutex_unlock(&conn->xmitmutex);
2597
2598         /*
2599          * Block until all in-progress commands for this connection
2600          * time out or fail.
2601          */
2602         for (;;) {
2603                 spin_lock_bh(&conn->lock);
2604                 if (!session->host->host_busy) { /* OK for ERL == 0 */
2605                         spin_unlock_bh(&conn->lock);
2606                         break;
2607                 }
2608                 spin_unlock_bh(&conn->lock);
2609                 msleep_interruptible(500);
2610                 printk("conn_destroy(): host_busy %d host_failed %d\n",
2611                         session->host->host_busy, session->host->host_failed);
2612                 /*
2613                  * force eh_abort() to unblock
2614                  */
2615                 wake_up(&conn->ehwait);
2616         }
2617
2618         /* now free crypto */
2619         if (conn->hdrdgst_en || conn->datadgst_en) {
2620                 if (conn->tx_tfm)
2621                         crypto_free_tfm(conn->tx_tfm);
2622                 if (conn->rx_tfm)
2623                         crypto_free_tfm(conn->rx_tfm);
2624                 if (conn->data_tx_tfm)
2625                         crypto_free_tfm(conn->data_tx_tfm);
2626                 if (conn->data_rx_tfm)
2627                         crypto_free_tfm(conn->data_rx_tfm);
2628         }
2629
2630         /* free conn->data, size = MaxRecvDataSegmentLength */
2631         if (conn->data_size <= PAGE_SIZE)
2632                 kfree(conn->data);
2633         else
2634                 free_pages((unsigned long)conn->data,
2635                                         get_order(conn->data_size));
2636
2637         spin_lock_bh(&session->lock);
2638         __kfifo_put(session->mgmtpool.queue, (void*)&conn->login_mtask,
2639                     sizeof(void*));
2640         list_del(&conn->item);
2641         if (list_empty(&session->connections))
2642                 session->leadconn = NULL;
2643         if (session->leadconn && session->leadconn == conn)
2644                 session->leadconn = container_of(session->connections.next,
2645                         struct iscsi_conn, item);
2646
2647         if (session->leadconn == NULL)
2648                 /* none connections exits.. reset sequencing */
2649                 session->cmdsn = session->max_cmdsn = session->exp_cmdsn = 1;
2650         spin_unlock_bh(&session->lock);
2651
2652         kfifo_free(conn->xmitqueue);
2653         kfifo_free(conn->writequeue);
2654         kfifo_free(conn->immqueue);
2655         kfifo_free(conn->mgmtqueue);
2656         kfree(conn);
2657 }
2658
2659 static int
2660 iscsi_conn_bind(iscsi_sessionh_t sessionh, iscsi_connh_t connh,
2661                 uint32_t transport_fd, int is_leading)
2662 {
2663         struct iscsi_session *session = iscsi_ptr(sessionh);
2664         struct iscsi_conn *tmp = ERR_PTR(-EEXIST), *conn = iscsi_ptr(connh);
2665         struct sock *sk;
2666         struct socket *sock;
2667         int err;
2668
2669         /* lookup for existing socket */
2670         sock = sockfd_lookup(transport_fd, &err);
2671         if (!sock) {
2672                 printk(KERN_ERR "iscsi_tcp: sockfd_lookup failed %d\n", err);
2673                 return -EEXIST;
2674         }
2675
2676         /* lookup for existing connection */
2677         spin_lock_bh(&session->lock);
2678         list_for_each_entry(tmp, &session->connections, item) {
2679                 if (tmp == conn) {
2680                         if (conn->c_stage != ISCSI_CONN_STOPPED ||
2681                             conn->stop_stage == STOP_CONN_TERM) {
2682                                 printk(KERN_ERR "iscsi_tcp: can't bind "
2683                                        "non-stopped connection (%d:%d)\n",
2684                                        conn->c_stage, conn->stop_stage);
2685                                 spin_unlock_bh(&session->lock);
2686                                 return -EIO;
2687                         }
2688                         break;
2689                 }
2690         }
2691         if (tmp != conn) {
2692                 /* bind new iSCSI connection to session */
2693                 conn->session = session;
2694
2695                 list_add(&conn->item, &session->connections);
2696         }
2697         spin_unlock_bh(&session->lock);
2698
2699         if (conn->stop_stage != STOP_CONN_SUSPEND) {
2700                 /* bind iSCSI connection and socket */
2701                 conn->sock = sock;
2702
2703                 /* setup Socket parameters */
2704                 sk = sock->sk;
2705                 sk->sk_reuse = 1;
2706                 sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
2707                 sk->sk_allocation = GFP_ATOMIC;
2708
2709                 /* FIXME: disable Nagle's algorithm */
2710
2711                 /*
2712                  * Intercept TCP callbacks for sendfile like receive
2713                  * processing.
2714                  */
2715                 iscsi_conn_set_callbacks(conn);
2716
2717                 /*
2718                  * set receive state machine into initial state
2719                  */
2720                 conn->in_progress = IN_PROGRESS_WAIT_HEADER;
2721         }
2722
2723         if (is_leading)
2724                 session->leadconn = conn;
2725
2726         /*
2727          * Unblock xmitworker(), Login Phase will pass through.
2728          */
2729         clear_bit(SUSPEND_BIT, &conn->suspend_rx);
2730         clear_bit(SUSPEND_BIT, &conn->suspend_tx);
2731
2732         return 0;
2733 }
2734
2735 static int
2736 iscsi_conn_start(iscsi_connh_t connh)
2737 {
2738         struct iscsi_conn *conn = iscsi_ptr(connh);
2739         struct iscsi_session *session = conn->session;
2740         struct sock *sk;
2741
2742         /* FF phase warming up... */
2743
2744         if (session == NULL) {
2745                 printk(KERN_ERR "iscsi_tcp: can't start unbound connection\n");
2746                 return -EPERM;
2747         }
2748
2749         sk = conn->sock->sk;
2750
2751         write_lock_bh(&sk->sk_callback_lock);
2752         spin_lock_bh(&session->lock);
2753         conn->c_stage = ISCSI_CONN_STARTED;
2754         session->state = ISCSI_STATE_LOGGED_IN;
2755
2756         switch(conn->stop_stage) {
2757         case STOP_CONN_RECOVER:
2758                 /*
2759                  * unblock eh_abort() if it is blocked. re-try all
2760                  * commands after successful recovery
2761                  */
2762                 session->conn_cnt++;
2763                 conn->stop_stage = 0;
2764                 conn->tmabort_state = TMABORT_INITIAL;
2765                 session->age++;
2766                 wake_up(&conn->ehwait);
2767                 break;
2768         case STOP_CONN_TERM:
2769                 session->conn_cnt++;
2770                 conn->stop_stage = 0;
2771                 break;
2772         case STOP_CONN_SUSPEND:
2773                 conn->stop_stage = 0;
2774                 clear_bit(SUSPEND_BIT, &conn->suspend_rx);
2775                 clear_bit(SUSPEND_BIT, &conn->suspend_tx);
2776                 break;
2777         default:
2778                 break;
2779         }
2780         spin_unlock_bh(&session->lock);
2781         write_unlock_bh(&sk->sk_callback_lock);
2782
2783         return 0;
2784 }
2785
2786 static void
2787 iscsi_conn_stop(iscsi_connh_t connh, int flag)
2788 {
2789         struct iscsi_conn *conn = iscsi_ptr(connh);
2790         struct iscsi_session *session = conn->session;
2791         struct sock *sk;
2792         unsigned long flags;
2793
2794         BUG_ON(!conn->sock);
2795         sk = conn->sock->sk;
2796         write_lock_bh(&sk->sk_callback_lock);
2797         set_bit(SUSPEND_BIT, &conn->suspend_rx);
2798         write_unlock_bh(&sk->sk_callback_lock);
2799
2800         mutex_lock(&conn->xmitmutex);
2801
2802         spin_lock_irqsave(session->host->host_lock, flags);
2803         spin_lock(&session->lock);
2804         conn->stop_stage = flag;
2805         conn->c_stage = ISCSI_CONN_STOPPED;
2806         set_bit(SUSPEND_BIT, &conn->suspend_tx);
2807
2808         if (flag != STOP_CONN_SUSPEND)
2809                 session->conn_cnt--;
2810
2811         if (session->conn_cnt == 0 || session->leadconn == conn)
2812                 session->state = ISCSI_STATE_FAILED;
2813
2814         spin_unlock(&session->lock);
2815         spin_unlock_irqrestore(session->host->host_lock, flags);
2816
2817         if (flag == STOP_CONN_TERM || flag == STOP_CONN_RECOVER) {
2818                 struct iscsi_cmd_task *ctask;
2819                 struct iscsi_mgmt_task *mtask;
2820
2821                 /*
2822                  * Socket must go now.
2823                  */
2824                 sock_hold(conn->sock->sk);
2825                 iscsi_conn_restore_callbacks(conn);
2826                 sock_put(conn->sock->sk);
2827
2828                 /*
2829                  * flush xmit queues.
2830                  */
2831                 spin_lock_bh(&session->lock);
2832                 while (__kfifo_get(conn->writequeue, (void*)&ctask,
2833                             sizeof(void*)) ||
2834                         __kfifo_get(conn->xmitqueue, (void*)&ctask,
2835                             sizeof(void*))) {
2836                         struct iscsi_r2t_info *r2t;
2837
2838                         /*
2839                          * flush ctask's r2t queues
2840                          */
2841                         while (__kfifo_get(ctask->r2tqueue, (void*)&r2t,
2842                                 sizeof(void*)))
2843                                 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t,
2844                                             sizeof(void*));
2845
2846                         spin_unlock_bh(&session->lock);
2847                         local_bh_disable();
2848                         iscsi_ctask_cleanup(conn, ctask);
2849                         local_bh_enable();
2850                         spin_lock_bh(&session->lock);
2851                 }
2852                 conn->ctask = NULL;
2853                 while (__kfifo_get(conn->immqueue, (void*)&mtask,
2854                            sizeof(void*)) ||
2855                         __kfifo_get(conn->mgmtqueue, (void*)&mtask,
2856                            sizeof(void*))) {
2857                         __kfifo_put(session->mgmtpool.queue,
2858                                     (void*)&mtask, sizeof(void*));
2859                 }
2860                 conn->mtask = NULL;
2861                 spin_unlock_bh(&session->lock);
2862
2863                 /*
2864                  * release socket only after we stopped data_xmit()
2865                  * activity and flushed all outstandings
2866                  */
2867                 sock_release(conn->sock);
2868                 conn->sock = NULL;
2869
2870                 /*
2871                  * for connection level recovery we should not calculate
2872                  * header digest. conn->hdr_size used for optimization
2873                  * in hdr_extract() and will be re-negotiated at
2874                  * set_param() time.
2875                  */
2876                 if (flag == STOP_CONN_RECOVER) {
2877                         conn->hdr_size = sizeof(struct iscsi_hdr);
2878                         conn->hdrdgst_en = 0;
2879                         conn->datadgst_en = 0;
2880                 }
2881         }
2882         mutex_unlock(&conn->xmitmutex);
2883 }
2884
2885 static int
2886 iscsi_conn_send_generic(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
2887                         char *data, uint32_t data_size)
2888 {
2889         struct iscsi_session *session = conn->session;
2890         struct iscsi_nopout *nop = (struct iscsi_nopout *)hdr;
2891         struct iscsi_mgmt_task *mtask;
2892
2893         spin_lock_bh(&session->lock);
2894         if (session->state == ISCSI_STATE_TERMINATE) {
2895                 spin_unlock_bh(&session->lock);
2896                 return -EPERM;
2897         }
2898         if (hdr->opcode == (ISCSI_OP_LOGIN | ISCSI_OP_IMMEDIATE) ||
2899             hdr->opcode == (ISCSI_OP_TEXT | ISCSI_OP_IMMEDIATE))
2900                 /*
2901                  * Login and Text are sent serially, in
2902                  * request-followed-by-response sequence.
2903                  * Same mtask can be used. Same ITT must be used.
2904                  * Note that login_mtask is preallocated at conn_create().
2905                  */
2906                 mtask = conn->login_mtask;
2907         else {
2908                 BUG_ON(conn->c_stage == ISCSI_CONN_INITIAL_STAGE);
2909                 BUG_ON(conn->c_stage == ISCSI_CONN_STOPPED);
2910
2911                 if (!__kfifo_get(session->mgmtpool.queue,
2912                                  (void*)&mtask, sizeof(void*))) {
2913                         spin_unlock_bh(&session->lock);
2914                         return -ENOSPC;
2915                 }
2916         }
2917
2918         /*
2919          * pre-format CmdSN and ExpStatSN for outgoing PDU.
2920          */
2921         if (hdr->itt != cpu_to_be32(ISCSI_RESERVED_TAG)) {
2922                 hdr->itt = mtask->itt | (conn->id << CID_SHIFT) |
2923                            (session->age << AGE_SHIFT);
2924                 nop->cmdsn = cpu_to_be32(session->cmdsn);
2925                 if (conn->c_stage == ISCSI_CONN_STARTED &&
2926                     !(hdr->opcode & ISCSI_OP_IMMEDIATE))
2927                         session->cmdsn++;
2928         } else
2929                 /* do not advance CmdSN */
2930                 nop->cmdsn = cpu_to_be32(session->cmdsn);
2931
2932         nop->exp_statsn = cpu_to_be32(conn->exp_statsn);
2933
2934         memcpy(&mtask->hdr, hdr, sizeof(struct iscsi_hdr));
2935
2936         iscsi_buf_init_virt(&mtask->headbuf, (char*)&mtask->hdr,
2937                                     sizeof(struct iscsi_hdr));
2938
2939         spin_unlock_bh(&session->lock);
2940
2941         if (data_size) {
2942                 memcpy(mtask->data, data, data_size);
2943                 mtask->data_count = data_size;
2944         } else
2945                 mtask->data_count = 0;
2946
2947         mtask->xmstate = XMSTATE_IMM_HDR;
2948
2949         if (mtask->data_count) {
2950                 iscsi_buf_init_iov(&mtask->sendbuf, (char*)mtask->data,
2951                                     mtask->data_count);
2952         }
2953
2954         debug_scsi("mgmtpdu [op 0x%x hdr->itt 0x%x datalen %d]\n",
2955                    hdr->opcode, hdr->itt, data_size);
2956
2957         /*
2958          * since send_pdu() could be called at least from two contexts,
2959          * we need to serialize __kfifo_put, so we don't have to take
2960          * additional lock on fast data-path
2961          */
2962         if (hdr->opcode & ISCSI_OP_IMMEDIATE)
2963                 __kfifo_put(conn->immqueue, (void*)&mtask, sizeof(void*));
2964         else
2965                 __kfifo_put(conn->mgmtqueue, (void*)&mtask, sizeof(void*));
2966
2967         schedule_work(&conn->xmitwork);
2968
2969         return 0;
2970 }
2971
2972 static int
2973 iscsi_eh_host_reset(struct scsi_cmnd *sc)
2974 {
2975         struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr;
2976         struct iscsi_conn *conn = ctask->conn;
2977         struct iscsi_session *session = conn->session;
2978
2979         spin_lock_bh(&session->lock);
2980         if (session->state == ISCSI_STATE_TERMINATE) {
2981                 debug_scsi("failing host reset: session terminated "
2982                            "[CID %d age %d]", conn->id, session->age);
2983                 spin_unlock_bh(&session->lock);
2984                 return FAILED;
2985         }
2986         spin_unlock_bh(&session->lock);
2987
2988         debug_scsi("failing connection CID %d due to SCSI host reset "
2989                    "[itt 0x%x age %d]", conn->id, ctask->itt,
2990                    session->age);
2991         iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
2992
2993         return SUCCESS;
2994 }
2995
2996 static void
2997 iscsi_tmabort_timedout(unsigned long data)
2998 {
2999         struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)data;
3000         struct iscsi_conn *conn = ctask->conn;
3001         struct iscsi_session *session = conn->session;
3002
3003         spin_lock(&session->lock);
3004         if (conn->tmabort_state == TMABORT_INITIAL) {
3005                 __kfifo_put(session->mgmtpool.queue,
3006                                 (void*)&ctask->mtask, sizeof(void*));
3007                 conn->tmabort_state = TMABORT_TIMEDOUT;
3008                 debug_scsi("tmabort timedout [sc %lx itt 0x%x]\n",
3009                         (long)ctask->sc, ctask->itt);
3010                 /* unblock eh_abort() */
3011                 wake_up(&conn->ehwait);
3012         }
3013         spin_unlock(&session->lock);
3014 }
3015
3016 static int
3017 iscsi_eh_abort(struct scsi_cmnd *sc)
3018 {
3019         int rc;
3020         struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr;
3021         struct iscsi_conn *conn = ctask->conn;
3022         struct iscsi_session *session = conn->session;
3023
3024         conn->eh_abort_cnt++;
3025         debug_scsi("aborting [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
3026
3027         /*
3028          * two cases for ERL=0 here:
3029          *
3030          * 1) connection-level failure;
3031          * 2) recovery due protocol error;
3032          */
3033         mutex_lock(&conn->xmitmutex);
3034         spin_lock_bh(&session->lock);
3035         if (session->state != ISCSI_STATE_LOGGED_IN) {
3036                 if (session->state == ISCSI_STATE_TERMINATE) {
3037                         spin_unlock_bh(&session->lock);
3038                         mutex_unlock(&conn->xmitmutex);
3039                         goto failed;
3040                 }
3041                 spin_unlock_bh(&session->lock);
3042         } else {
3043                 struct iscsi_tm *hdr = &conn->tmhdr;
3044
3045                 /*
3046                  * Still LOGGED_IN...
3047                  */
3048
3049                 if (!ctask->sc || sc->SCp.phase != session->age) {
3050                         /*
3051                          * 1) ctask completed before time out. But session
3052                          *    is still ok => Happy Retry.
3053                          * 2) session was re-open during time out of ctask.
3054                          */
3055                         spin_unlock_bh(&session->lock);
3056                         mutex_unlock(&conn->xmitmutex);
3057                         goto success;
3058                 }
3059                 conn->tmabort_state = TMABORT_INITIAL;
3060                 spin_unlock_bh(&session->lock);
3061
3062                 /*
3063                  * ctask timed out but session is OK
3064                  * ERL=0 requires task mgmt abort to be issued on each
3065                  * failed command. requests must be serialized.
3066                  */
3067                 memset(hdr, 0, sizeof(struct iscsi_tm));
3068                 hdr->opcode = ISCSI_OP_SCSI_TMFUNC | ISCSI_OP_IMMEDIATE;
3069                 hdr->flags = ISCSI_TM_FUNC_ABORT_TASK;
3070                 hdr->flags |= ISCSI_FLAG_CMD_FINAL;
3071                 memcpy(hdr->lun, ctask->hdr.lun, sizeof(hdr->lun));
3072                 hdr->rtt = ctask->hdr.itt;
3073                 hdr->refcmdsn = ctask->hdr.cmdsn;
3074
3075                 rc = iscsi_conn_send_generic(conn, (struct iscsi_hdr *)hdr,
3076                                              NULL, 0);
3077                 if (rc) {
3078                         iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3079                         debug_scsi("abort sent failure [itt 0x%x]", ctask->itt);
3080                 } else {
3081                         struct iscsi_r2t_info *r2t;
3082
3083                         /*
3084                          * TMF abort vs. TMF response race logic
3085                          */
3086                         spin_lock_bh(&session->lock);
3087                         ctask->mtask = (struct iscsi_mgmt_task *)
3088                                 session->mgmt_cmds[(hdr->itt & ITT_MASK) -
3089                                                         ISCSI_MGMT_ITT_OFFSET];
3090                         /*
3091                          * have to flush r2tqueue to avoid r2t leaks
3092                          */
3093                         while (__kfifo_get(ctask->r2tqueue, (void*)&r2t,
3094                                 sizeof(void*))) {
3095                                 __kfifo_put(ctask->r2tpool.queue, (void*)&r2t,
3096                                         sizeof(void*));
3097                         }
3098                         if (conn->tmabort_state == TMABORT_INITIAL) {
3099                                 conn->tmfcmd_pdus_cnt++;
3100                                 conn->tmabort_timer.expires = 3*HZ + jiffies;
3101                                 conn->tmabort_timer.function =
3102                                                 iscsi_tmabort_timedout;
3103                                 conn->tmabort_timer.data = (unsigned long)ctask;
3104                                 add_timer(&conn->tmabort_timer);
3105                                 debug_scsi("abort sent [itt 0x%x]", ctask->itt);
3106                         } else {
3107                                 if (!ctask->sc ||
3108                                     conn->tmabort_state == TMABORT_SUCCESS) {
3109                                         conn->tmabort_state = TMABORT_INITIAL;
3110                                         spin_unlock_bh(&session->lock);
3111                                         mutex_unlock(&conn->xmitmutex);
3112                                         goto success;
3113                                 }
3114                                 conn->tmabort_state = TMABORT_INITIAL;
3115                                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3116                         }
3117                         spin_unlock_bh(&session->lock);
3118                 }
3119         }
3120         mutex_unlock(&conn->xmitmutex);
3121
3122
3123         /*
3124          * block eh thread until:
3125          *
3126          * 1) abort response;
3127          * 2) abort timeout;
3128          * 3) session re-opened;
3129          * 4) session terminated;
3130          */
3131         for (;;) {
3132                 int p_state = session->state;
3133
3134                 rc = wait_event_interruptible(conn->ehwait,
3135                         (p_state == ISCSI_STATE_LOGGED_IN ?
3136                          (session->state == ISCSI_STATE_TERMINATE ||
3137                           conn->tmabort_state != TMABORT_INITIAL) :
3138                          (session->state == ISCSI_STATE_TERMINATE ||
3139                           session->state == ISCSI_STATE_LOGGED_IN)));
3140                 if (rc) {
3141                         /* shutdown.. */
3142                         session->state = ISCSI_STATE_TERMINATE;
3143                         goto failed;
3144                 }
3145
3146                 if (signal_pending(current))
3147                         flush_signals(current);
3148
3149                 if (session->state == ISCSI_STATE_TERMINATE)
3150                         goto failed;
3151
3152                 spin_lock_bh(&session->lock);
3153                 if (sc->SCp.phase == session->age &&
3154                    (conn->tmabort_state == TMABORT_TIMEDOUT ||
3155                     conn->tmabort_state == TMABORT_FAILED)) {
3156                         conn->tmabort_state = TMABORT_INITIAL;
3157                         if (!ctask->sc) {
3158                                 /*
3159                                  * ctask completed before tmf abort response or
3160                                  * time out.
3161                                  * But session is still ok => Happy Retry.
3162                                  */
3163                                 spin_unlock_bh(&session->lock);
3164                                 break;
3165                         }
3166                         spin_unlock_bh(&session->lock);
3167                         iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
3168                         continue;
3169                 }
3170                 spin_unlock_bh(&session->lock);
3171                 break;
3172         }
3173
3174 success:
3175         debug_scsi("abort success [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
3176         rc = SUCCESS;
3177         goto exit;
3178
3179 failed:
3180         debug_scsi("abort failed [sc %lx itt 0x%x]\n", (long)sc, ctask->itt);
3181         rc = FAILED;
3182
3183 exit:
3184         del_timer_sync(&conn->tmabort_timer);
3185
3186         mutex_lock(&conn->xmitmutex);
3187         if (conn->sock) {
3188                 struct sock *sk = conn->sock->sk;
3189
3190                 write_lock_bh(&sk->sk_callback_lock);
3191                 iscsi_ctask_cleanup(conn, ctask);
3192                 write_unlock_bh(&sk->sk_callback_lock);
3193         }
3194         mutex_unlock(&conn->xmitmutex);
3195         return rc;
3196 }
3197
3198 static int
3199 iscsi_r2tpool_alloc(struct iscsi_session *session)
3200 {
3201         int i;
3202         int cmd_i;
3203
3204         /*
3205          * initialize per-task: R2T pool and xmit queue
3206          */
3207         for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
3208                 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
3209
3210                 /*
3211                  * pre-allocated x4 as much r2ts to handle race when
3212                  * target acks DataOut faster than we data_xmit() queues
3213                  * could replenish r2tqueue.
3214                  */
3215
3216                 /* R2T pool */
3217                 if (iscsi_pool_init(&ctask->r2tpool, session->max_r2t * 4,
3218                         (void***)&ctask->r2ts, sizeof(struct iscsi_r2t_info))) {
3219                         goto r2t_alloc_fail;
3220                 }
3221
3222                 /* R2T xmit queue */
3223                 ctask->r2tqueue = kfifo_alloc(
3224                       session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL);
3225                 if (ctask->r2tqueue == ERR_PTR(-ENOMEM)) {
3226                         iscsi_pool_free(&ctask->r2tpool, (void**)ctask->r2ts);
3227                         goto r2t_alloc_fail;
3228                 }
3229
3230                 /*
3231                  * number of
3232                  * Data-Out PDU's within R2T-sequence can be quite big;
3233                  * using mempool
3234                  */
3235                 ctask->datapool = mempool_create(ISCSI_DTASK_DEFAULT_MAX,
3236                          mempool_alloc_slab, mempool_free_slab, taskcache);
3237                 if (ctask->datapool == NULL) {
3238                         kfifo_free(ctask->r2tqueue);
3239                         iscsi_pool_free(&ctask->r2tpool, (void**)ctask->r2ts);
3240                         goto r2t_alloc_fail;
3241                 }
3242                 INIT_LIST_HEAD(&ctask->dataqueue);
3243         }
3244
3245         return 0;
3246
3247 r2t_alloc_fail:
3248         for (i = 0; i < cmd_i; i++) {
3249                 mempool_destroy(session->cmds[i]->datapool);
3250                 kfifo_free(session->cmds[i]->r2tqueue);
3251                 iscsi_pool_free(&session->cmds[i]->r2tpool,
3252                                 (void**)session->cmds[i]->r2ts);
3253         }
3254         return -ENOMEM;
3255 }
3256
3257 static void
3258 iscsi_r2tpool_free(struct iscsi_session *session)
3259 {
3260         int i;
3261
3262         for (i = 0; i < session->cmds_max; i++) {
3263                 mempool_destroy(session->cmds[i]->datapool);
3264                 kfifo_free(session->cmds[i]->r2tqueue);
3265                 iscsi_pool_free(&session->cmds[i]->r2tpool,
3266                                 (void**)session->cmds[i]->r2ts);
3267         }
3268 }
3269
3270 static struct scsi_host_template iscsi_sht = {
3271         .name                   = "iSCSI Initiator over TCP/IP, v."
3272                                   ISCSI_VERSION_STR,
3273         .queuecommand           = iscsi_queuecommand,
3274         .change_queue_depth     = iscsi_change_queue_depth,
3275         .can_queue              = ISCSI_XMIT_CMDS_MAX - 1,
3276         .sg_tablesize           = ISCSI_SG_TABLESIZE,
3277         .cmd_per_lun            = ISCSI_DEF_CMD_PER_LUN,
3278         .eh_abort_handler       = iscsi_eh_abort,
3279         .eh_host_reset_handler  = iscsi_eh_host_reset,
3280         .use_clustering         = DISABLE_CLUSTERING,
3281         .proc_name              = "iscsi_tcp",
3282         .this_id                = -1,
3283 };
3284
3285 static iscsi_sessionh_t
3286 iscsi_session_create(uint32_t initial_cmdsn, struct Scsi_Host *host)
3287 {
3288         int cmd_i;
3289         struct iscsi_session *session;
3290
3291         session = iscsi_hostdata(host->hostdata);
3292         memset(session, 0, sizeof(struct iscsi_session));
3293
3294         session->host = host;
3295         session->id = host->host_no;
3296         session->state = ISCSI_STATE_LOGGED_IN;
3297         session->mgmtpool_max = ISCSI_MGMT_CMDS_MAX;
3298         session->cmds_max = ISCSI_XMIT_CMDS_MAX;
3299         session->cmdsn = initial_cmdsn;
3300         session->exp_cmdsn = initial_cmdsn + 1;
3301         session->max_cmdsn = initial_cmdsn + 1;
3302         session->max_r2t = 1;
3303
3304         /* initialize SCSI PDU commands pool */
3305         if (iscsi_pool_init(&session->cmdpool, session->cmds_max,
3306                 (void***)&session->cmds, sizeof(struct iscsi_cmd_task)))
3307                 goto cmdpool_alloc_fail;
3308
3309         /* pre-format cmds pool with ITT */
3310         for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++)
3311                 session->cmds[cmd_i]->itt = cmd_i;
3312
3313         spin_lock_init(&session->lock);
3314         INIT_LIST_HEAD(&session->connections);
3315
3316         /* initialize immediate command pool */
3317         if (iscsi_pool_init(&session->mgmtpool, session->mgmtpool_max,
3318                 (void***)&session->mgmt_cmds, sizeof(struct iscsi_mgmt_task)))
3319                 goto mgmtpool_alloc_fail;
3320
3321
3322         /* pre-format immediate cmds pool with ITT */
3323         for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++) {
3324                 session->mgmt_cmds[cmd_i]->itt = ISCSI_MGMT_ITT_OFFSET + cmd_i;
3325                 session->mgmt_cmds[cmd_i]->data = kmalloc(
3326                         DEFAULT_MAX_RECV_DATA_SEGMENT_LENGTH, GFP_KERNEL);
3327                 if (!session->mgmt_cmds[cmd_i]->data) {
3328                         int j;
3329
3330                         for (j = 0; j < cmd_i; j++)
3331                                 kfree(session->mgmt_cmds[j]->data);
3332                         goto immdata_alloc_fail;
3333                 }
3334         }
3335
3336         if (iscsi_r2tpool_alloc(session))
3337                 goto r2tpool_alloc_fail;
3338
3339         return iscsi_handle(session);
3340
3341 r2tpool_alloc_fail:
3342         for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++)
3343                 kfree(session->mgmt_cmds[cmd_i]->data);
3344         iscsi_pool_free(&session->mgmtpool, (void**)session->mgmt_cmds);
3345 immdata_alloc_fail:
3346 mgmtpool_alloc_fail:
3347         iscsi_pool_free(&session->cmdpool, (void**)session->cmds);
3348 cmdpool_alloc_fail:
3349         return iscsi_handle(NULL);
3350 }
3351
3352 static void
3353 iscsi_session_destroy(iscsi_sessionh_t sessionh)
3354 {
3355         int cmd_i;
3356         struct iscsi_data_task *dtask, *n;
3357         struct iscsi_session *session = iscsi_ptr(sessionh);
3358
3359         for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
3360                 struct iscsi_cmd_task *ctask = session->cmds[cmd_i];
3361                 list_for_each_entry_safe(dtask, n, &ctask->dataqueue, item) {
3362                         list_del(&dtask->item);
3363                         mempool_free(dtask, ctask->datapool);
3364                 }
3365         }
3366
3367         for (cmd_i = 0; cmd_i < session->mgmtpool_max; cmd_i++)
3368                 kfree(session->mgmt_cmds[cmd_i]->data);
3369
3370         iscsi_r2tpool_free(session);
3371         iscsi_pool_free(&session->mgmtpool, (void**)session->mgmt_cmds);
3372         iscsi_pool_free(&session->cmdpool, (void**)session->cmds);
3373 }
3374
3375 static int
3376 iscsi_conn_set_param(iscsi_connh_t connh, enum iscsi_param param,
3377                      uint32_t value)
3378 {
3379         struct iscsi_conn *conn = iscsi_ptr(connh);
3380         struct iscsi_session *session = conn->session;
3381
3382         spin_lock_bh(&session->lock);
3383         if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE &&
3384             conn->stop_stage != STOP_CONN_RECOVER) {
3385                 printk(KERN_ERR "iscsi_tcp: can not change parameter [%d]\n",
3386                        param);
3387                 spin_unlock_bh(&session->lock);
3388                 return 0;
3389         }
3390         spin_unlock_bh(&session->lock);
3391
3392         switch(param) {
3393         case ISCSI_PARAM_MAX_RECV_DLENGTH: {
3394                 char *saveptr = conn->data;
3395                 gfp_t flags = GFP_KERNEL;
3396
3397                 if (conn->data_size >= value) {
3398                         conn->max_recv_dlength = value;
3399                         break;
3400                 }
3401
3402                 spin_lock_bh(&session->lock);
3403                 if (conn->stop_stage == STOP_CONN_RECOVER)
3404                         flags = GFP_ATOMIC;
3405                 spin_unlock_bh(&session->lock);
3406
3407                 if (value <= PAGE_SIZE)
3408                         conn->data = kmalloc(value, flags);
3409                 else
3410                         conn->data = (void*)__get_free_pages(flags,
3411                                                              get_order(value));
3412                 if (conn->data == NULL) {
3413                         conn->data = saveptr;
3414                         return -ENOMEM;
3415                 }
3416                 if (conn->data_size <= PAGE_SIZE)
3417                         kfree(saveptr);
3418                 else
3419                         free_pages((unsigned long)saveptr,
3420                                    get_order(conn->data_size));
3421                 conn->max_recv_dlength = value;
3422                 conn->data_size = value;
3423                 }
3424                 break;
3425         case ISCSI_PARAM_MAX_XMIT_DLENGTH:
3426                 conn->max_xmit_dlength =  value;
3427                 break;
3428         case ISCSI_PARAM_HDRDGST_EN:
3429                 conn->hdrdgst_en = value;
3430                 conn->hdr_size = sizeof(struct iscsi_hdr);
3431                 if (conn->hdrdgst_en) {
3432                         conn->hdr_size += sizeof(__u32);
3433                         if (!conn->tx_tfm)
3434                                 conn->tx_tfm = crypto_alloc_tfm("crc32c", 0);
3435                         if (!conn->tx_tfm)
3436                                 return -ENOMEM;
3437                         if (!conn->rx_tfm)
3438                                 conn->rx_tfm = crypto_alloc_tfm("crc32c", 0);
3439                         if (!conn->rx_tfm) {
3440                                 crypto_free_tfm(conn->tx_tfm);
3441                                 return -ENOMEM;
3442                         }
3443                 } else {
3444                         if (conn->tx_tfm)
3445                                 crypto_free_tfm(conn->tx_tfm);
3446                         if (conn->rx_tfm)
3447                                 crypto_free_tfm(conn->rx_tfm);
3448                 }
3449                 break;
3450         case ISCSI_PARAM_DATADGST_EN:
3451                 conn->datadgst_en = value;
3452                 if (conn->datadgst_en) {
3453                         if (!conn->data_tx_tfm)
3454                                 conn->data_tx_tfm =
3455                                     crypto_alloc_tfm("crc32c", 0);
3456                         if (!conn->data_tx_tfm)
3457                                 return -ENOMEM;
3458                         if (!conn->data_rx_tfm)
3459                                 conn->data_rx_tfm =
3460                                     crypto_alloc_tfm("crc32c", 0);
3461                         if (!conn->data_rx_tfm) {
3462                                 crypto_free_tfm(conn->data_tx_tfm);
3463                                 return -ENOMEM;
3464                         }
3465                 } else {
3466                         if (conn->data_tx_tfm)
3467                                 crypto_free_tfm(conn->data_tx_tfm);
3468                         if (conn->data_rx_tfm)
3469                                 crypto_free_tfm(conn->data_rx_tfm);
3470                 }
3471                 break;
3472         case ISCSI_PARAM_INITIAL_R2T_EN:
3473                 session->initial_r2t_en = value;
3474                 break;
3475         case ISCSI_PARAM_MAX_R2T:
3476                 if (session->max_r2t == roundup_pow_of_two(value))
3477                         break;
3478                 iscsi_r2tpool_free(session);
3479                 session->max_r2t = value;
3480                 if (session->max_r2t & (session->max_r2t - 1))
3481                         session->max_r2t = roundup_pow_of_two(session->max_r2t);
3482                 if (iscsi_r2tpool_alloc(session))
3483                         return -ENOMEM;
3484                 break;
3485         case ISCSI_PARAM_IMM_DATA_EN:
3486                 session->imm_data_en = value;
3487                 break;
3488         case ISCSI_PARAM_FIRST_BURST:
3489                 session->first_burst = value;
3490                 break;
3491         case ISCSI_PARAM_MAX_BURST:
3492                 session->max_burst = value;
3493                 break;
3494         case ISCSI_PARAM_PDU_INORDER_EN:
3495                 session->pdu_inorder_en = value;
3496                 break;
3497         case ISCSI_PARAM_DATASEQ_INORDER_EN:
3498                 session->dataseq_inorder_en = value;
3499                 break;
3500         case ISCSI_PARAM_ERL:
3501                 session->erl = value;
3502                 break;
3503         case ISCSI_PARAM_IFMARKER_EN:
3504                 BUG_ON(value);
3505                 session->ifmarker_en = value;
3506                 break;
3507         case ISCSI_PARAM_OFMARKER_EN:
3508                 BUG_ON(value);
3509                 session->ofmarker_en = value;
3510                 break;
3511         default:
3512                 break;
3513         }
3514
3515         return 0;
3516 }
3517
3518 static int
3519 iscsi_conn_get_param(iscsi_connh_t connh, enum iscsi_param param,
3520                      uint32_t *value)
3521 {
3522         struct iscsi_conn *conn = iscsi_ptr(connh);
3523         struct iscsi_session *session = conn->session;
3524
3525         switch(param) {
3526         case ISCSI_PARAM_MAX_RECV_DLENGTH:
3527                 *value = conn->max_recv_dlength;
3528                 break;
3529         case ISCSI_PARAM_MAX_XMIT_DLENGTH:
3530                 *value = conn->max_xmit_dlength;
3531                 break;
3532         case ISCSI_PARAM_HDRDGST_EN:
3533                 *value = conn->hdrdgst_en;
3534                 break;
3535         case ISCSI_PARAM_DATADGST_EN:
3536                 *value = conn->datadgst_en;
3537                 break;
3538         case ISCSI_PARAM_INITIAL_R2T_EN:
3539                 *value = session->initial_r2t_en;
3540                 break;
3541         case ISCSI_PARAM_MAX_R2T:
3542                 *value = session->max_r2t;
3543                 break;
3544         case ISCSI_PARAM_IMM_DATA_EN:
3545                 *value = session->imm_data_en;
3546                 break;
3547         case ISCSI_PARAM_FIRST_BURST:
3548                 *value = session->first_burst;
3549                 break;
3550         case ISCSI_PARAM_MAX_BURST:
3551                 *value = session->max_burst;
3552                 break;
3553         case ISCSI_PARAM_PDU_INORDER_EN:
3554                 *value = session->pdu_inorder_en;
3555                 break;
3556         case ISCSI_PARAM_DATASEQ_INORDER_EN:
3557                 *value = session->dataseq_inorder_en;
3558                 break;
3559         case ISCSI_PARAM_ERL:
3560                 *value = session->erl;
3561                 break;
3562         case ISCSI_PARAM_IFMARKER_EN:
3563                 *value = session->ifmarker_en;
3564                 break;
3565         case ISCSI_PARAM_OFMARKER_EN:
3566                 *value = session->ofmarker_en;
3567                 break;
3568         default:
3569                 return ISCSI_ERR_PARAM_NOT_FOUND;
3570         }
3571
3572         return 0;
3573 }
3574
3575 static void
3576 iscsi_conn_get_stats(iscsi_connh_t connh, struct iscsi_stats *stats)
3577 {
3578         struct iscsi_conn *conn = iscsi_ptr(connh);
3579
3580         stats->txdata_octets = conn->txdata_octets;
3581         stats->rxdata_octets = conn->rxdata_octets;
3582         stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
3583         stats->dataout_pdus = conn->dataout_pdus_cnt;
3584         stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
3585         stats->datain_pdus = conn->datain_pdus_cnt;
3586         stats->r2t_pdus = conn->r2t_pdus_cnt;
3587         stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
3588         stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
3589         stats->custom_length = 3;
3590         strcpy(stats->custom[0].desc, "tx_sendpage_failures");
3591         stats->custom[0].value = conn->sendpage_failures_cnt;
3592         strcpy(stats->custom[1].desc, "rx_discontiguous_hdr");
3593         stats->custom[1].value = conn->discontiguous_hdr_cnt;
3594         strcpy(stats->custom[2].desc, "eh_abort_cnt");
3595         stats->custom[2].value = conn->eh_abort_cnt;
3596 }
3597
3598 static int
3599 iscsi_conn_send_pdu(iscsi_connh_t connh, struct iscsi_hdr *hdr, char *data,
3600                     uint32_t data_size)
3601 {
3602         struct iscsi_conn *conn = iscsi_ptr(connh);
3603         int rc;
3604
3605         mutex_lock(&conn->xmitmutex);
3606         rc = iscsi_conn_send_generic(conn, hdr, data, data_size);
3607         mutex_unlock(&conn->xmitmutex);
3608
3609         return rc;
3610 }
3611
3612 static struct iscsi_transport iscsi_tcp_transport = {
3613         .owner                  = THIS_MODULE,
3614         .name                   = "tcp",
3615         .caps                   = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
3616                                   | CAP_DATADGST,
3617         .host_template          = &iscsi_sht,
3618         .hostdata_size          = sizeof(struct iscsi_session),
3619         .max_conn               = 1,
3620         .max_cmd_len            = ISCSI_TCP_MAX_CMD_LEN,
3621         .create_session         = iscsi_session_create,
3622         .destroy_session        = iscsi_session_destroy,
3623         .create_conn            = iscsi_conn_create,
3624         .bind_conn              = iscsi_conn_bind,
3625         .destroy_conn           = iscsi_conn_destroy,
3626         .set_param              = iscsi_conn_set_param,
3627         .get_param              = iscsi_conn_get_param,
3628         .start_conn             = iscsi_conn_start,
3629         .stop_conn              = iscsi_conn_stop,
3630         .send_pdu               = iscsi_conn_send_pdu,
3631         .get_stats              = iscsi_conn_get_stats,
3632 };
3633
3634 static int __init
3635 iscsi_tcp_init(void)
3636 {
3637         int error;
3638
3639         if (iscsi_max_lun < 1) {
3640                 printk(KERN_ERR "Invalid max_lun value of %u\n", iscsi_max_lun);
3641                 return -EINVAL;
3642         }
3643         iscsi_tcp_transport.max_lun = iscsi_max_lun;
3644
3645         taskcache = kmem_cache_create("iscsi_taskcache",
3646                         sizeof(struct iscsi_data_task), 0,
3647                         SLAB_HWCACHE_ALIGN | SLAB_NO_REAP, NULL, NULL);
3648         if (!taskcache)
3649                 return -ENOMEM;
3650
3651         error = iscsi_register_transport(&iscsi_tcp_transport);
3652         if (error)
3653                 kmem_cache_destroy(taskcache);
3654
3655         return error;
3656 }
3657
3658 static void __exit
3659 iscsi_tcp_exit(void)
3660 {
3661         iscsi_unregister_transport(&iscsi_tcp_transport);
3662         kmem_cache_destroy(taskcache);
3663 }
3664
3665 module_init(iscsi_tcp_init);
3666 module_exit(iscsi_tcp_exit);