drivers/target/iscsi/iscsi_target_erl0.c

   1 /******************************************************************************
   2  * This file contains error recovery level zero functions used by
   3  * the iSCSI Target driver.
   4  *
   5  * (c) Copyright 2007-2013 Datera, Inc.
   6  *
   7  * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  ******************************************************************************/
  19
  20 #include <scsi/iscsi_proto.h>
  21 #include <target/target_core_base.h>
  22 #include <target/target_core_fabric.h>
  23
  24 #include "iscsi_target_core.h"
  25 #include "iscsi_target_seq_pdu_list.h"
  26 #include "iscsi_target_tq.h"
  27 #include "iscsi_target_erl0.h"
  28 #include "iscsi_target_erl1.h"
  29 #include "iscsi_target_erl2.h"
  30 #include "iscsi_target_util.h"
  31 #include "iscsi_target.h"
  32
  33 /*
  34  *      Used to set values in struct iscsi_cmd that iscsit_dataout_check_sequence()
  35  *      checks against to determine a PDU's Offset+Length is within the current
  36  *      DataOUT Sequence.  Used for DataSequenceInOrder=Yes only.
  37  */
  38 void iscsit_set_dataout_sequence_values(
  39         struct iscsi_cmd *cmd)
  40 {
  41         struct iscsi_conn *conn = cmd->conn;
  42         /*
  43          * Still set seq_start_offset and seq_end_offset for Unsolicited
  44          * DataOUT, even if DataSequenceInOrder=No.
  45          */
  46         if (cmd->unsolicited_data) {
  47                 cmd->seq_start_offset = cmd->write_data_done;
  48                 cmd->seq_end_offset = (cmd->write_data_done +
  49                         ((cmd->se_cmd.data_length >
  50                           conn->sess->sess_ops->FirstBurstLength) ?
  51                          conn->sess->sess_ops->FirstBurstLength : cmd->se_cmd.data_length));
  52                 return;
  53         }
  54
  55         if (!conn->sess->sess_ops->DataSequenceInOrder)
  56                 return;
  57
  58         if (!cmd->seq_start_offset && !cmd->seq_end_offset) {
  59                 cmd->seq_start_offset = cmd->write_data_done;
  60                 cmd->seq_end_offset = (cmd->se_cmd.data_length >
  61                         conn->sess->sess_ops->MaxBurstLength) ?
  62                         (cmd->write_data_done +
  63                         conn->sess->sess_ops->MaxBurstLength) : cmd->se_cmd.data_length;
  64         } else {
  65                 cmd->seq_start_offset = cmd->seq_end_offset;
  66                 cmd->seq_end_offset = ((cmd->seq_end_offset +
  67                         conn->sess->sess_ops->MaxBurstLength) >=
  68                         cmd->se_cmd.data_length) ? cmd->se_cmd.data_length :
  69                         (cmd->seq_end_offset +
  70                          conn->sess->sess_ops->MaxBurstLength);
  71         }
  72 }
  73
  74 static int iscsit_dataout_within_command_recovery_check(
  75         struct iscsi_cmd *cmd,
  76         unsigned char *buf)
  77 {
  78         struct iscsi_conn *conn = cmd->conn;
  79         struct iscsi_data *hdr = (struct iscsi_data *) buf;
  80         u32 payload_length = ntoh24(hdr->dlength);
  81
  82         /*
  83          * We do the within-command recovery checks here as it is
  84          * the first function called in iscsi_check_pre_dataout().
  85          * Basically, if we are in within-command recovery and
  86          * the PDU does not contain the offset the sequence needs,
  87          * dump the payload.
  88          *
  89          * This only applies to DataPDUInOrder=Yes, for
  90          * DataPDUInOrder=No we only re-request the failed PDU
  91          * and check that all PDUs in a sequence are received
  92          * upon end of sequence.
  93          */
  94         if (conn->sess->sess_ops->DataSequenceInOrder) {
  95                 if ((cmd->cmd_flags & ICF_WITHIN_COMMAND_RECOVERY) &&
  96                     cmd->write_data_done != be32_to_cpu(hdr->offset))
  97                         goto dump;
  98
  99                 cmd->cmd_flags &= ~ICF_WITHIN_COMMAND_RECOVERY;
 100         } else {
 101                 struct iscsi_seq *seq;
 102
 103                 seq = iscsit_get_seq_holder(cmd, be32_to_cpu(hdr->offset),
 104                                             payload_length);
 105                 if (!seq)
 106                         return DATAOUT_CANNOT_RECOVER;
 107                 /*
 108                  * Set the struct iscsi_seq pointer to reuse later.
 109                  */
 110                 cmd->seq_ptr = seq;
 111
 112                 if (conn->sess->sess_ops->DataPDUInOrder) {
 113                         if (seq->status ==
 114                             DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY &&
 115                            (seq->offset != be32_to_cpu(hdr->offset) ||
 116                             seq->data_sn != be32_to_cpu(hdr->datasn)))
 117                                 goto dump;
 118                 } else {
 119                         if (seq->status ==
 120                              DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY &&
 121                             seq->data_sn != be32_to_cpu(hdr->datasn))
 122                                 goto dump;
 123                 }
 124
 125                 if (seq->status == DATAOUT_SEQUENCE_COMPLETE)
 126                         goto dump;
 127
 128                 if (seq->status != DATAOUT_SEQUENCE_COMPLETE)
 129                         seq->status = 0;
 130         }
 131
 132         return DATAOUT_NORMAL;
 133
 134 dump:
 135         pr_err("Dumping DataOUT PDU Offset: %u Length: %d DataSN:"
 136                 " 0x%08x\n", hdr->offset, payload_length, hdr->datasn);
 137         return iscsit_dump_data_payload(conn, payload_length, 1);
 138 }
 139
 140 static int iscsit_dataout_check_unsolicited_sequence(
 141         struct iscsi_cmd *cmd,
 142         unsigned char *buf)
 143 {
 144         u32 first_burst_len;
 145         struct iscsi_conn *conn = cmd->conn;
 146         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 147         u32 payload_length = ntoh24(hdr->dlength);
 148
 149
 150         if ((be32_to_cpu(hdr->offset) < cmd->seq_start_offset) ||
 151            ((be32_to_cpu(hdr->offset) + payload_length) > cmd->seq_end_offset)) {
 152                 pr_err("Command ITT: 0x%08x with Offset: %u,"
 153                 " Length: %u outside of Unsolicited Sequence %u:%u while"
 154                 " DataSequenceInOrder=Yes.\n", cmd->init_task_tag,
 155                 be32_to_cpu(hdr->offset), payload_length, cmd->seq_start_offset,
 156                         cmd->seq_end_offset);
 157                 return DATAOUT_CANNOT_RECOVER;
 158         }
 159
 160         first_burst_len = (cmd->first_burst_len + payload_length);
 161
 162         if (first_burst_len > conn->sess->sess_ops->FirstBurstLength) {
 163                 pr_err("Total %u bytes exceeds FirstBurstLength: %u"
 164                         " for this Unsolicited DataOut Burst.\n",
 165                         first_burst_len, conn->sess->sess_ops->FirstBurstLength);
 166                 transport_send_check_condition_and_sense(&cmd->se_cmd,
 167                                 TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 168                 return DATAOUT_CANNOT_RECOVER;
 169         }
 170
 171         /*
 172          * Perform various MaxBurstLength and ISCSI_FLAG_CMD_FINAL sanity
 173          * checks for the current Unsolicited DataOUT Sequence.
 174          */
 175         if (hdr->flags & ISCSI_FLAG_CMD_FINAL) {
 176                 /*
 177                  * Ignore ISCSI_FLAG_CMD_FINAL checks while DataPDUInOrder=No, end of
 178                  * sequence checks are handled in
 179                  * iscsit_dataout_datapduinorder_no_fbit().
 180                  */
 181                 if (!conn->sess->sess_ops->DataPDUInOrder)
 182                         goto out;
 183
 184                 if ((first_burst_len != cmd->se_cmd.data_length) &&
 185                     (first_burst_len != conn->sess->sess_ops->FirstBurstLength)) {
 186                         pr_err("Unsolicited non-immediate data"
 187                         " received %u does not equal FirstBurstLength: %u, and"
 188                         " does not equal ExpXferLen %u.\n", first_burst_len,
 189                                 conn->sess->sess_ops->FirstBurstLength,
 190                                 cmd->se_cmd.data_length);
 191                         transport_send_check_condition_and_sense(&cmd->se_cmd,
 192                                         TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 193                         return DATAOUT_CANNOT_RECOVER;
 194                 }
 195         } else {
 196                 if (first_burst_len == conn->sess->sess_ops->FirstBurstLength) {
 197                         pr_err("Command ITT: 0x%08x reached"
 198                         " FirstBurstLength: %u, but ISCSI_FLAG_CMD_FINAL is not set. protocol"
 199                                 " error.\n", cmd->init_task_tag,
 200                                 conn->sess->sess_ops->FirstBurstLength);
 201                         return DATAOUT_CANNOT_RECOVER;
 202                 }
 203                 if (first_burst_len == cmd->se_cmd.data_length) {
 204                         pr_err("Command ITT: 0x%08x reached"
 205                         " ExpXferLen: %u, but ISCSI_FLAG_CMD_FINAL is not set. protocol"
 206                         " error.\n", cmd->init_task_tag, cmd->se_cmd.data_length);
 207                         return DATAOUT_CANNOT_RECOVER;
 208                 }
 209         }
 210
 211 out:
 212         return DATAOUT_NORMAL;
 213 }
 214
 215 static int iscsit_dataout_check_sequence(
 216         struct iscsi_cmd *cmd,
 217         unsigned char *buf)
 218 {
 219         u32 next_burst_len;
 220         struct iscsi_conn *conn = cmd->conn;
 221         struct iscsi_seq *seq = NULL;
 222         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 223         u32 payload_length = ntoh24(hdr->dlength);
 224
 225         /*
 226          * For DataSequenceInOrder=Yes: Check that the offset and offset+length
 227          * is within range as defined by iscsi_set_dataout_sequence_values().
 228          *
 229          * For DataSequenceInOrder=No: Check that an struct iscsi_seq exists for
 230          * offset+length tuple.
 231          */
 232         if (conn->sess->sess_ops->DataSequenceInOrder) {
 233                 /*
 234                  * Due to possibility of recovery DataOUT sent by the initiator
 235                  * fullfilling an Recovery R2T, it's best to just dump the
 236                  * payload here, instead of erroring out.
 237                  */
 238                 if ((be32_to_cpu(hdr->offset) < cmd->seq_start_offset) ||
 239                    ((be32_to_cpu(hdr->offset) + payload_length) > cmd->seq_end_offset)) {
 240                         pr_err("Command ITT: 0x%08x with Offset: %u,"
 241                         " Length: %u outside of Sequence %u:%u while"
 242                         " DataSequenceInOrder=Yes.\n", cmd->init_task_tag,
 243                         be32_to_cpu(hdr->offset), payload_length, cmd->seq_start_offset,
 244                                 cmd->seq_end_offset);
 245
 246                         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 247                                 return DATAOUT_CANNOT_RECOVER;
 248                         return DATAOUT_WITHIN_COMMAND_RECOVERY;
 249                 }
 250
 251                 next_burst_len = (cmd->next_burst_len + payload_length);
 252         } else {
 253                 seq = iscsit_get_seq_holder(cmd, be32_to_cpu(hdr->offset),
 254                                             payload_length);
 255                 if (!seq)
 256                         return DATAOUT_CANNOT_RECOVER;
 257                 /*
 258                  * Set the struct iscsi_seq pointer to reuse later.
 259                  */
 260                 cmd->seq_ptr = seq;
 261
 262                 if (seq->status == DATAOUT_SEQUENCE_COMPLETE) {
 263                         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 264                                 return DATAOUT_CANNOT_RECOVER;
 265                         return DATAOUT_WITHIN_COMMAND_RECOVERY;
 266                 }
 267
 268                 next_burst_len = (seq->next_burst_len + payload_length);
 269         }
 270
 271         if (next_burst_len > conn->sess->sess_ops->MaxBurstLength) {
 272                 pr_err("Command ITT: 0x%08x, NextBurstLength: %u and"
 273                         " Length: %u exceeds MaxBurstLength: %u. protocol"
 274                         " error.\n", cmd->init_task_tag,
 275                         (next_burst_len - payload_length),
 276                         payload_length, conn->sess->sess_ops->MaxBurstLength);
 277                 return DATAOUT_CANNOT_RECOVER;
 278         }
 279
 280         /*
 281          * Perform various MaxBurstLength and ISCSI_FLAG_CMD_FINAL sanity
 282          * checks for the current DataOUT Sequence.
 283          */
 284         if (hdr->flags & ISCSI_FLAG_CMD_FINAL) {
 285                 /*
 286                  * Ignore ISCSI_FLAG_CMD_FINAL checks while DataPDUInOrder=No, end of
 287                  * sequence checks are handled in
 288                  * iscsit_dataout_datapduinorder_no_fbit().
 289                  */
 290                 if (!conn->sess->sess_ops->DataPDUInOrder)
 291                         goto out;
 292
 293                 if (conn->sess->sess_ops->DataSequenceInOrder) {
 294                         if ((next_burst_len <
 295                              conn->sess->sess_ops->MaxBurstLength) &&
 296                            ((cmd->write_data_done + payload_length) <
 297                              cmd->se_cmd.data_length)) {
 298                                 pr_err("Command ITT: 0x%08x set ISCSI_FLAG_CMD_FINAL"
 299                                 " before end of DataOUT sequence, protocol"
 300                                 " error.\n", cmd->init_task_tag);
 301                                 return DATAOUT_CANNOT_RECOVER;
 302                         }
 303                 } else {
 304                         if (next_burst_len < seq->xfer_len) {
 305                                 pr_err("Command ITT: 0x%08x set ISCSI_FLAG_CMD_FINAL"
 306                                 " before end of DataOUT sequence, protocol"
 307                                 " error.\n", cmd->init_task_tag);
 308                                 return DATAOUT_CANNOT_RECOVER;
 309                         }
 310                 }
 311         } else {
 312                 if (conn->sess->sess_ops->DataSequenceInOrder) {
 313                         if (next_burst_len ==
 314                                         conn->sess->sess_ops->MaxBurstLength) {
 315                                 pr_err("Command ITT: 0x%08x reached"
 316                                 " MaxBurstLength: %u, but ISCSI_FLAG_CMD_FINAL is"
 317                                 " not set, protocol error.", cmd->init_task_tag,
 318                                         conn->sess->sess_ops->MaxBurstLength);
 319                                 return DATAOUT_CANNOT_RECOVER;
 320                         }
 321                         if ((cmd->write_data_done + payload_length) ==
 322                                         cmd->se_cmd.data_length) {
 323                                 pr_err("Command ITT: 0x%08x reached"
 324                                 " last DataOUT PDU in sequence but ISCSI_FLAG_"
 325                                 "CMD_FINAL is not set, protocol error.\n",
 326                                         cmd->init_task_tag);
 327                                 return DATAOUT_CANNOT_RECOVER;
 328                         }
 329                 } else {
 330                         if (next_burst_len == seq->xfer_len) {
 331                                 pr_err("Command ITT: 0x%08x reached"
 332                                 " last DataOUT PDU in sequence but ISCSI_FLAG_"
 333                                 "CMD_FINAL is not set, protocol error.\n",
 334                                         cmd->init_task_tag);
 335                                 return DATAOUT_CANNOT_RECOVER;
 336                         }
 337                 }
 338         }
 339
 340 out:
 341         return DATAOUT_NORMAL;
 342 }
 343
 344 static int iscsit_dataout_check_datasn(
 345         struct iscsi_cmd *cmd,
 346         unsigned char *buf)
 347 {
 348         int dump = 0, recovery = 0;
 349         u32 data_sn = 0;
 350         struct iscsi_conn *conn = cmd->conn;
 351         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 352         u32 payload_length = ntoh24(hdr->dlength);
 353
 354         /*
 355          * Considering the target has no method of re-requesting DataOUT
 356          * by DataSN, if we receieve a greater DataSN than expected we
 357          * assume the functions for DataPDUInOrder=[Yes,No] below will
 358          * handle it.
 359          *
 360          * If the DataSN is less than expected, dump the payload.
 361          */
 362         if (conn->sess->sess_ops->DataSequenceInOrder)
 363                 data_sn = cmd->data_sn;
 364         else {
 365                 struct iscsi_seq *seq = cmd->seq_ptr;
 366                 data_sn = seq->data_sn;
 367         }
 368
 369         if (be32_to_cpu(hdr->datasn) > data_sn) {
 370                 pr_err("Command ITT: 0x%08x, received DataSN: 0x%08x"
 371                         " higher than expected 0x%08x.\n", cmd->init_task_tag,
 372                                 be32_to_cpu(hdr->datasn), data_sn);
 373                 recovery = 1;
 374                 goto recover;
 375         } else if (be32_to_cpu(hdr->datasn) < data_sn) {
 376                 pr_err("Command ITT: 0x%08x, received DataSN: 0x%08x"
 377                         " lower than expected 0x%08x, discarding payload.\n",
 378                         cmd->init_task_tag, be32_to_cpu(hdr->datasn), data_sn);
 379                 dump = 1;
 380                 goto dump;
 381         }
 382
 383         return DATAOUT_NORMAL;
 384
 385 recover:
 386         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 387                 pr_err("Unable to perform within-command recovery"
 388                                 " while ERL=0.\n");
 389                 return DATAOUT_CANNOT_RECOVER;
 390         }
 391 dump:
 392         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 393                 return DATAOUT_CANNOT_RECOVER;
 394
 395         return (recovery || dump) ? DATAOUT_WITHIN_COMMAND_RECOVERY :
 396                                 DATAOUT_NORMAL;
 397 }
 398
 399 static int iscsit_dataout_pre_datapduinorder_yes(
 400         struct iscsi_cmd *cmd,
 401         unsigned char *buf)
 402 {
 403         int dump = 0, recovery = 0;
 404         struct iscsi_conn *conn = cmd->conn;
 405         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 406         u32 payload_length = ntoh24(hdr->dlength);
 407
 408         /*
 409          * For DataSequenceInOrder=Yes: If the offset is greater than the global
 410          * DataPDUInOrder=Yes offset counter in struct iscsi_cmd a protcol error has
 411          * occurred and fail the connection.
 412          *
 413          * For DataSequenceInOrder=No: If the offset is greater than the per
 414          * sequence DataPDUInOrder=Yes offset counter in struct iscsi_seq a protocol
 415          * error has occurred and fail the connection.
 416          */
 417         if (conn->sess->sess_ops->DataSequenceInOrder) {
 418                 if (be32_to_cpu(hdr->offset) != cmd->write_data_done) {
 419                         pr_err("Command ITT: 0x%08x, received offset"
 420                         " %u different than expected %u.\n", cmd->init_task_tag,
 421                                 be32_to_cpu(hdr->offset), cmd->write_data_done);
 422                         recovery = 1;
 423                         goto recover;
 424                 }
 425         } else {
 426                 struct iscsi_seq *seq = cmd->seq_ptr;
 427
 428                 if (be32_to_cpu(hdr->offset) > seq->offset) {
 429                         pr_err("Command ITT: 0x%08x, received offset"
 430                         " %u greater than expected %u.\n", cmd->init_task_tag,
 431                                 be32_to_cpu(hdr->offset), seq->offset);
 432                         recovery = 1;
 433                         goto recover;
 434                 } else if (be32_to_cpu(hdr->offset) < seq->offset) {
 435                         pr_err("Command ITT: 0x%08x, received offset"
 436                         " %u less than expected %u, discarding payload.\n",
 437                                 cmd->init_task_tag, be32_to_cpu(hdr->offset),
 438                                 seq->offset);
 439                         dump = 1;
 440                         goto dump;
 441                 }
 442         }
 443
 444         return DATAOUT_NORMAL;
 445
 446 recover:
 447         if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 448                 pr_err("Unable to perform within-command recovery"
 449                                 " while ERL=0.\n");
 450                 return DATAOUT_CANNOT_RECOVER;
 451         }
 452 dump:
 453         if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 454                 return DATAOUT_CANNOT_RECOVER;
 455
 456         return (recovery) ? iscsit_recover_dataout_sequence(cmd,
 457                 be32_to_cpu(hdr->offset), payload_length) :
 458                (dump) ? DATAOUT_WITHIN_COMMAND_RECOVERY : DATAOUT_NORMAL;
 459 }
 460
 461 static int iscsit_dataout_pre_datapduinorder_no(
 462         struct iscsi_cmd *cmd,
 463         unsigned char *buf)
 464 {
 465         struct iscsi_pdu *pdu;
 466         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 467         u32 payload_length = ntoh24(hdr->dlength);
 468
 469         pdu = iscsit_get_pdu_holder(cmd, be32_to_cpu(hdr->offset),
 470                                     payload_length);
 471         if (!pdu)
 472                 return DATAOUT_CANNOT_RECOVER;
 473
 474         cmd->pdu_ptr = pdu;
 475
 476         switch (pdu->status) {
 477         case ISCSI_PDU_NOT_RECEIVED:
 478         case ISCSI_PDU_CRC_FAILED:
 479         case ISCSI_PDU_TIMED_OUT:
 480                 break;
 481         case ISCSI_PDU_RECEIVED_OK:
 482                 pr_err("Command ITT: 0x%08x received already gotten"
 483                         " Offset: %u, Length: %u\n", cmd->init_task_tag,
 484                                 be32_to_cpu(hdr->offset), payload_length);
 485                 return iscsit_dump_data_payload(cmd->conn, payload_length, 1);
 486         default:
 487                 return DATAOUT_CANNOT_RECOVER;
 488         }
 489
 490         return DATAOUT_NORMAL;
 491 }
 492
 493 static int iscsit_dataout_update_r2t(struct iscsi_cmd *cmd, u32 offset, u32 length)
 494 {
 495         struct iscsi_r2t *r2t;
 496
 497         if (cmd->unsolicited_data)
 498                 return 0;
 499
 500         r2t = iscsit_get_r2t_for_eos(cmd, offset, length);
 501         if (!r2t)
 502                 return -1;
 503
 504         spin_lock_bh(&cmd->r2t_lock);
 505         r2t->seq_complete = 1;
 506         cmd->outstanding_r2ts--;
 507         spin_unlock_bh(&cmd->r2t_lock);
 508
 509         return 0;
 510 }
 511
 512 static int iscsit_dataout_update_datapduinorder_no(
 513         struct iscsi_cmd *cmd,
 514         u32 data_sn,
 515         int f_bit)
 516 {
 517         int ret = 0;
 518         struct iscsi_pdu *pdu = cmd->pdu_ptr;
 519
 520         pdu->data_sn = data_sn;
 521
 522         switch (pdu->status) {
 523         case ISCSI_PDU_NOT_RECEIVED:
 524                 pdu->status = ISCSI_PDU_RECEIVED_OK;
 525                 break;
 526         case ISCSI_PDU_CRC_FAILED:
 527                 pdu->status = ISCSI_PDU_RECEIVED_OK;
 528                 break;
 529         case ISCSI_PDU_TIMED_OUT:
 530                 pdu->status = ISCSI_PDU_RECEIVED_OK;
 531                 break;
 532         default:
 533                 return DATAOUT_CANNOT_RECOVER;
 534         }
 535
 536         if (f_bit) {
 537                 ret = iscsit_dataout_datapduinorder_no_fbit(cmd, pdu);
 538                 if (ret == DATAOUT_CANNOT_RECOVER)
 539                         return ret;
 540         }
 541
 542         return DATAOUT_NORMAL;
 543 }
 544
 545 static int iscsit_dataout_post_crc_passed(
 546         struct iscsi_cmd *cmd,
 547         unsigned char *buf)
 548 {
 549         int ret, send_r2t = 0;
 550         struct iscsi_conn *conn = cmd->conn;
 551         struct iscsi_seq *seq = NULL;
 552         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 553         u32 payload_length = ntoh24(hdr->dlength);
 554
 555         if (cmd->unsolicited_data) {
 556                 if ((cmd->first_burst_len + payload_length) ==
 557                      conn->sess->sess_ops->FirstBurstLength) {
 558                         if (iscsit_dataout_update_r2t(cmd, be32_to_cpu(hdr->offset),
 559                                         payload_length) < 0)
 560                                 return DATAOUT_CANNOT_RECOVER;
 561                         send_r2t = 1;
 562                 }
 563
 564                 if (!conn->sess->sess_ops->DataPDUInOrder) {
 565                         ret = iscsit_dataout_update_datapduinorder_no(cmd,
 566                                 be32_to_cpu(hdr->datasn),
 567                                 (hdr->flags & ISCSI_FLAG_CMD_FINAL));
 568                         if (ret == DATAOUT_CANNOT_RECOVER)
 569                                 return ret;
 570                 }
 571
 572                 cmd->first_burst_len += payload_length;
 573
 574                 if (conn->sess->sess_ops->DataSequenceInOrder)
 575                         cmd->data_sn++;
 576                 else {
 577                         seq = cmd->seq_ptr;
 578                         seq->data_sn++;
 579                         seq->offset += payload_length;
 580                 }
 581
 582                 if (send_r2t) {
 583                         if (seq)
 584                                 seq->status = DATAOUT_SEQUENCE_COMPLETE;
 585                         cmd->first_burst_len = 0;
 586                         cmd->unsolicited_data = 0;
 587                 }
 588         } else {
 589                 if (conn->sess->sess_ops->DataSequenceInOrder) {
 590                         if ((cmd->next_burst_len + payload_length) ==
 591                              conn->sess->sess_ops->MaxBurstLength) {
 592                                 if (iscsit_dataout_update_r2t(cmd,
 593                                                 be32_to_cpu(hdr->offset),
 594                                                 payload_length) < 0)
 595                                         return DATAOUT_CANNOT_RECOVER;
 596                                 send_r2t = 1;
 597                         }
 598
 599                         if (!conn->sess->sess_ops->DataPDUInOrder) {
 600                                 ret = iscsit_dataout_update_datapduinorder_no(
 601                                                 cmd, be32_to_cpu(hdr->datasn),
 602                                                 (hdr->flags & ISCSI_FLAG_CMD_FINAL));
 603                                 if (ret == DATAOUT_CANNOT_RECOVER)
 604                                         return ret;
 605                         }
 606
 607                         cmd->next_burst_len += payload_length;
 608                         cmd->data_sn++;
 609
 610                         if (send_r2t)
 611                                 cmd->next_burst_len = 0;
 612                 } else {
 613                         seq = cmd->seq_ptr;
 614
 615                         if ((seq->next_burst_len + payload_length) ==
 616                              seq->xfer_len) {
 617                                 if (iscsit_dataout_update_r2t(cmd,
 618                                                 be32_to_cpu(hdr->offset),
 619                                                 payload_length) < 0)
 620                                         return DATAOUT_CANNOT_RECOVER;
 621                                 send_r2t = 1;
 622                         }
 623
 624                         if (!conn->sess->sess_ops->DataPDUInOrder) {
 625                                 ret = iscsit_dataout_update_datapduinorder_no(
 626                                                 cmd, be32_to_cpu(hdr->datasn),
 627                                                 (hdr->flags & ISCSI_FLAG_CMD_FINAL));
 628                                 if (ret == DATAOUT_CANNOT_RECOVER)
 629                                         return ret;
 630                         }
 631
 632                         seq->data_sn++;
 633                         seq->offset += payload_length;
 634                         seq->next_burst_len += payload_length;
 635
 636                         if (send_r2t) {
 637                                 seq->next_burst_len = 0;
 638                                 seq->status = DATAOUT_SEQUENCE_COMPLETE;
 639                         }
 640                 }
 641         }
 642
 643         if (send_r2t && conn->sess->sess_ops->DataSequenceInOrder)
 644                 cmd->data_sn = 0;
 645
 646         cmd->write_data_done += payload_length;
 647
 648         if (cmd->write_data_done == cmd->se_cmd.data_length)
 649                 return DATAOUT_SEND_TO_TRANSPORT;
 650         else if (send_r2t)
 651                 return DATAOUT_SEND_R2T;
 652         else
 653                 return DATAOUT_NORMAL;
 654 }
 655
 656 static int iscsit_dataout_post_crc_failed(
 657         struct iscsi_cmd *cmd,
 658         unsigned char *buf)
 659 {
 660         struct iscsi_conn *conn = cmd->conn;
 661         struct iscsi_pdu *pdu;
 662         struct iscsi_data *hdr = (struct iscsi_data *) buf;
 663         u32 payload_length = ntoh24(hdr->dlength);
 664
 665         if (conn->sess->sess_ops->DataPDUInOrder)
 666                 goto recover;
 667         /*
 668          * The rest of this function is only called when DataPDUInOrder=No.
 669          */
 670         pdu = cmd->pdu_ptr;
 671
 672         switch (pdu->status) {
 673         case ISCSI_PDU_NOT_RECEIVED:
 674                 pdu->status = ISCSI_PDU_CRC_FAILED;
 675                 break;
 676         case ISCSI_PDU_CRC_FAILED:
 677                 break;
 678         case ISCSI_PDU_TIMED_OUT:
 679                 pdu->status = ISCSI_PDU_CRC_FAILED;
 680                 break;
 681         default:
 682                 return DATAOUT_CANNOT_RECOVER;
 683         }
 684
 685 recover:
 686         return iscsit_recover_dataout_sequence(cmd, be32_to_cpu(hdr->offset),
 687                                                 payload_length);
 688 }
 689
 690 /*
 691  *      Called from iscsit_handle_data_out() before DataOUT Payload is received
 692  *      and CRC computed.
 693  */
 694 int iscsit_check_pre_dataout(
 695         struct iscsi_cmd *cmd,
 696         unsigned char *buf)
 697 {
 698         int ret;
 699         struct iscsi_conn *conn = cmd->conn;
 700
 701         ret = iscsit_dataout_within_command_recovery_check(cmd, buf);
 702         if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 703             (ret == DATAOUT_CANNOT_RECOVER))
 704                 return ret;
 705
 706         ret = iscsit_dataout_check_datasn(cmd, buf);
 707         if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 708             (ret == DATAOUT_CANNOT_RECOVER))
 709                 return ret;
 710
 711         if (cmd->unsolicited_data) {
 712                 ret = iscsit_dataout_check_unsolicited_sequence(cmd, buf);
 713                 if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 714                     (ret == DATAOUT_CANNOT_RECOVER))
 715                         return ret;
 716         } else {
 717                 ret = iscsit_dataout_check_sequence(cmd, buf);
 718                 if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 719                     (ret == DATAOUT_CANNOT_RECOVER))
 720                         return ret;
 721         }
 722
 723         return (conn->sess->sess_ops->DataPDUInOrder) ?
 724                 iscsit_dataout_pre_datapduinorder_yes(cmd, buf) :
 725                 iscsit_dataout_pre_datapduinorder_no(cmd, buf);
 726 }
 727
 728 /*
 729  *      Called from iscsit_handle_data_out() after DataOUT Payload is received
 730  *      and CRC computed.
 731  */
 732 int iscsit_check_post_dataout(
 733         struct iscsi_cmd *cmd,
 734         unsigned char *buf,
 735         u8 data_crc_failed)
 736 {
 737         struct iscsi_conn *conn = cmd->conn;
 738
 739         cmd->dataout_timeout_retries = 0;
 740
 741         if (!data_crc_failed)
 742                 return iscsit_dataout_post_crc_passed(cmd, buf);
 743         else {
 744                 if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 745                         pr_err("Unable to recover from DataOUT CRC"
 746                                 " failure while ERL=0, closing session.\n");
 747                         iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR,
 748                                           buf);
 749                         return DATAOUT_CANNOT_RECOVER;
 750                 }
 751
 752                 iscsit_reject_cmd(cmd, ISCSI_REASON_DATA_DIGEST_ERROR, buf);
 753                 return iscsit_dataout_post_crc_failed(cmd, buf);
 754         }
 755 }
 756
 757 static void iscsit_handle_time2retain_timeout(unsigned long data)
 758 {
 759         struct iscsi_session *sess = (struct iscsi_session *) data;
 760         struct iscsi_portal_group *tpg = sess->tpg;
 761         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
 762
 763         spin_lock_bh(&se_tpg->session_lock);
 764         if (sess->time2retain_timer_flags & ISCSI_TF_STOP) {
 765                 spin_unlock_bh(&se_tpg->session_lock);
 766                 return;
 767         }
 768         if (atomic_read(&sess->session_reinstatement)) {
 769                 pr_err("Exiting Time2Retain handler because"
 770                                 " session_reinstatement=1\n");
 771                 spin_unlock_bh(&se_tpg->session_lock);
 772                 return;
 773         }
 774         sess->time2retain_timer_flags |= ISCSI_TF_EXPIRED;
 775
 776         pr_err("Time2Retain timer expired for SID: %u, cleaning up"
 777                         " iSCSI session.\n", sess->sid);
 778         {
 779         struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
 780
 781         if (tiqn) {
 782                 spin_lock(&tiqn->sess_err_stats.lock);
 783                 strcpy(tiqn->sess_err_stats.last_sess_fail_rem_name,
 784                         (void *)sess->sess_ops->InitiatorName);
 785                 tiqn->sess_err_stats.last_sess_failure_type =
 786                                 ISCSI_SESS_ERR_CXN_TIMEOUT;
 787                 tiqn->sess_err_stats.cxn_timeout_errors++;
 788                 atomic_long_inc(&sess->conn_timeout_errors);
 789                 spin_unlock(&tiqn->sess_err_stats.lock);
 790         }
 791         }
 792
 793         spin_unlock_bh(&se_tpg->session_lock);
 794         target_put_session(sess->se_sess);
 795 }
 796
 797 void iscsit_start_time2retain_handler(struct iscsi_session *sess)
 798 {
 799         int tpg_active;
 800         /*
 801          * Only start Time2Retain timer when the associated TPG is still in
 802          * an ACTIVE (eg: not disabled or shutdown) state.
 803          */
 804         spin_lock(&sess->tpg->tpg_state_lock);
 805         tpg_active = (sess->tpg->tpg_state == TPG_STATE_ACTIVE);
 806         spin_unlock(&sess->tpg->tpg_state_lock);
 807
 808         if (!tpg_active)
 809                 return;
 810
 811         if (sess->time2retain_timer_flags & ISCSI_TF_RUNNING)
 812                 return;
 813
 814         pr_debug("Starting Time2Retain timer for %u seconds on"
 815                 " SID: %u\n", sess->sess_ops->DefaultTime2Retain, sess->sid);
 816
 817         init_timer(&sess->time2retain_timer);
 818         sess->time2retain_timer.expires =
 819                 (get_jiffies_64() + sess->sess_ops->DefaultTime2Retain * HZ);
 820         sess->time2retain_timer.data = (unsigned long)sess;
 821         sess->time2retain_timer.function = iscsit_handle_time2retain_timeout;
 822         sess->time2retain_timer_flags &= ~ISCSI_TF_STOP;
 823         sess->time2retain_timer_flags |= ISCSI_TF_RUNNING;
 824         add_timer(&sess->time2retain_timer);
 825 }
 826
 827 /*
 828  *      Called with spin_lock_bh(&struct se_portal_group->session_lock) held
 829  */
 830 int iscsit_stop_time2retain_timer(struct iscsi_session *sess)
 831 {
 832         struct iscsi_portal_group *tpg = sess->tpg;
 833         struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
 834
 835         if (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)
 836                 return -1;
 837
 838         if (!(sess->time2retain_timer_flags & ISCSI_TF_RUNNING))
 839                 return 0;
 840
 841         sess->time2retain_timer_flags |= ISCSI_TF_STOP;
 842         spin_unlock(&se_tpg->session_lock);
 843
 844         del_timer_sync(&sess->time2retain_timer);
 845
 846         spin_lock(&se_tpg->session_lock);
 847         sess->time2retain_timer_flags &= ~ISCSI_TF_RUNNING;
 848         pr_debug("Stopped Time2Retain Timer for SID: %u\n",
 849                         sess->sid);
 850         return 0;
 851 }
 852
 853 void iscsit_connection_reinstatement_rcfr(struct iscsi_conn *conn)
 854 {
 855         spin_lock_bh(&conn->state_lock);
 856         if (atomic_read(&conn->connection_exit)) {
 857                 spin_unlock_bh(&conn->state_lock);
 858                 goto sleep;
 859         }
 860
 861         if (atomic_read(&conn->transport_failed)) {
 862                 spin_unlock_bh(&conn->state_lock);
 863                 goto sleep;
 864         }
 865         spin_unlock_bh(&conn->state_lock);
 866
 867         iscsi_thread_set_force_reinstatement(conn);
 868
 869 sleep:
 870         wait_for_completion(&conn->conn_wait_rcfr_comp);
 871         complete(&conn->conn_post_wait_comp);
 872 }
 873
 874 void iscsit_cause_connection_reinstatement(struct iscsi_conn *conn, int sleep)
 875 {
 876         spin_lock_bh(&conn->state_lock);
 877         if (atomic_read(&conn->connection_exit)) {
 878                 spin_unlock_bh(&conn->state_lock);
 879                 return;
 880         }
 881
 882         if (atomic_read(&conn->transport_failed)) {
 883                 spin_unlock_bh(&conn->state_lock);
 884                 return;
 885         }
 886
 887         if (atomic_read(&conn->connection_reinstatement)) {
 888                 spin_unlock_bh(&conn->state_lock);
 889                 return;
 890         }
 891
 892         if (iscsi_thread_set_force_reinstatement(conn) < 0) {
 893                 spin_unlock_bh(&conn->state_lock);
 894                 return;
 895         }
 896
 897         atomic_set(&conn->connection_reinstatement, 1);
 898         if (!sleep) {
 899                 spin_unlock_bh(&conn->state_lock);
 900                 return;
 901         }
 902
 903         atomic_set(&conn->sleep_on_conn_wait_comp, 1);
 904         spin_unlock_bh(&conn->state_lock);
 905
 906         wait_for_completion(&conn->conn_wait_comp);
 907         complete(&conn->conn_post_wait_comp);
 908 }
 909 EXPORT_SYMBOL(iscsit_cause_connection_reinstatement);
 910
 911 void iscsit_fall_back_to_erl0(struct iscsi_session *sess)
 912 {
 913         pr_debug("Falling back to ErrorRecoveryLevel=0 for SID:"
 914                         " %u\n", sess->sid);
 915
 916         atomic_set(&sess->session_fall_back_to_erl0, 1);
 917 }
 918
 919 static void iscsit_handle_connection_cleanup(struct iscsi_conn *conn)
 920 {
 921         struct iscsi_session *sess = conn->sess;
 922
 923         if ((sess->sess_ops->ErrorRecoveryLevel == 2) &&
 924             !atomic_read(&sess->session_reinstatement) &&
 925             !atomic_read(&sess->session_fall_back_to_erl0))
 926                 iscsit_connection_recovery_transport_reset(conn);
 927         else {
 928                 pr_debug("Performing cleanup for failed iSCSI"
 929                         " Connection ID: %hu from %s\n", conn->cid,
 930                         sess->sess_ops->InitiatorName);
 931                 iscsit_close_connection(conn);
 932         }
 933 }
 934
 935 void iscsit_take_action_for_connection_exit(struct iscsi_conn *conn)
 936 {
 937         spin_lock_bh(&conn->state_lock);
 938         if (atomic_read(&conn->connection_exit)) {
 939                 spin_unlock_bh(&conn->state_lock);
 940                 return;
 941         }
 942         atomic_set(&conn->connection_exit, 1);
 943
 944         if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) {
 945                 spin_unlock_bh(&conn->state_lock);
 946                 iscsit_close_connection(conn);
 947                 return;
 948         }
 949
 950         if (conn->conn_state == TARG_CONN_STATE_CLEANUP_WAIT) {
 951                 spin_unlock_bh(&conn->state_lock);
 952                 return;
 953         }
 954
 955         pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n");
 956         conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT;
 957         spin_unlock_bh(&conn->state_lock);
 958
 959         iscsit_handle_connection_cleanup(conn);
 960 }
 961
 962 /*
 963  *      This is the simple function that makes the magic of
 964  *      sync and steering happen in the follow paradoxical order:
 965  *
 966  *      0) Receive conn->of_marker (bytes left until next OFMarker)
 967  *         bytes into an offload buffer.  When we pass the exact number
 968  *         of bytes in conn->of_marker, iscsit_dump_data_payload() and hence
 969  *         rx_data() will automatically receive the identical u32 marker
 970  *         values and store it in conn->of_marker_offset;
 971  *      1) Now conn->of_marker_offset will contain the offset to the start
 972  *         of the next iSCSI PDU.  Dump these remaining bytes into another
 973  *         offload buffer.
 974  *      2) We are done!
 975  *         Next byte in the TCP stream will contain the next iSCSI PDU!
 976  *         Cool Huh?!
 977  */
 978 int iscsit_recover_from_unknown_opcode(struct iscsi_conn *conn)
 979 {
 980         /*
 981          * Make sure the remaining bytes to next maker is a sane value.
 982          */
 983         if (conn->of_marker > (conn->conn_ops->OFMarkInt * 4)) {
 984                 pr_err("Remaining bytes to OFMarker: %u exceeds"
 985                         " OFMarkInt bytes: %u.\n", conn->of_marker,
 986                                 conn->conn_ops->OFMarkInt * 4);
 987                 return -1;
 988         }
 989
 990         pr_debug("Advancing %u bytes in TCP stream to get to the"
 991                         " next OFMarker.\n", conn->of_marker);
 992
 993         if (iscsit_dump_data_payload(conn, conn->of_marker, 0) < 0)
 994                 return -1;
 995
 996         /*
 997          * Make sure the offset marker we retrived is a valid value.
 998          */
 999         if (conn->of_marker_offset > (ISCSI_HDR_LEN + (ISCSI_CRC_LEN * 2) +
1000             conn->conn_ops->MaxRecvDataSegmentLength)) {
1001                 pr_err("OfMarker offset value: %u exceeds limit.\n",
1002                         conn->of_marker_offset);
1003                 return -1;
1004         }
1005
1006         pr_debug("Discarding %u bytes of TCP stream to get to the"
1007                         " next iSCSI Opcode.\n", conn->of_marker_offset);
1008
1009         if (iscsit_dump_data_payload(conn, conn->of_marker_offset, 0) < 0)
1010                 return -1;
1011
1012         return 0;
1013 }