SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/scsi/iscsi_tcp.c

   1 /*
   2  * iSCSI Initiator over TCP/IP Data-Path
   3  *
   4  * Copyright (C) 2004 Dmitry Yusupov
   5  * Copyright (C) 2004 Alex Aizman
   6  * Copyright (C) 2005 - 2006 Mike Christie
   7  * Copyright (C) 2006 Red Hat, Inc.  All rights reserved.
   8  * maintained by open-iscsi@googlegroups.com
   9  *
  10  * This program is free software; you can redistribute it and/or modify
  11  * it under the terms of the GNU General Public License as published
  12  * by the Free Software Foundation; either version 2 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful, but
  16  * WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  18  * General Public License for more details.
  19  *
  20  * See the file COPYING included with this distribution for more details.
  21  *
  22  * Credits:
  23  *      Christoph Hellwig
  24  *      FUJITA Tomonori
  25  *      Arne Redlich
  26  *      Zhenyu Wang
  27  */
  28
  29 #include <linux/types.h>
  30 #include <linux/list.h>
  31 #include <linux/inet.h>
  32 #include <linux/file.h>
  33 #include <linux/blkdev.h>
  34 #include <linux/crypto.h>
  35 #include <linux/delay.h>
  36 #include <linux/kfifo.h>
  37 #include <linux/scatterlist.h>
  38 #include <net/tcp.h>
  39 #include <scsi/scsi_cmnd.h>
  40 #include <scsi/scsi_device.h>
  41 #include <scsi/scsi_host.h>
  42 #include <scsi/scsi.h>
  43 #include <scsi/scsi_transport_iscsi.h>
  44
  45 #include "iscsi_tcp.h"
  46
  47 MODULE_AUTHOR("Dmitry Yusupov <dmitry_yus@yahoo.com>, "
  48               "Alex Aizman <itn780@yahoo.com>");
  49 MODULE_DESCRIPTION("iSCSI/TCP data-path");
  50 MODULE_LICENSE("GPL");
  51 #undef DEBUG_TCP
  52 #define DEBUG_ASSERT
  53
  54 #ifdef DEBUG_TCP
  55 #define debug_tcp(fmt...) printk(KERN_INFO "tcp: " fmt)
  56 #else
  57 #define debug_tcp(fmt...)
  58 #endif
  59
  60 #ifndef DEBUG_ASSERT
  61 #ifdef BUG_ON
  62 #undef BUG_ON
  63 #endif
  64 #define BUG_ON(expr)
  65 #endif
  66
  67 static struct scsi_transport_template *iscsi_tcp_scsi_transport;
  68 static struct scsi_host_template iscsi_sht;
  69 static struct iscsi_transport iscsi_tcp_transport;
  70
  71 static unsigned int iscsi_max_lun = 512;
  72 module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
  73
  74 static int iscsi_tcp_hdr_recv_done(struct iscsi_tcp_conn *tcp_conn,
  75                                    struct iscsi_segment *segment);
  76
  77 /*
  78  * Scatterlist handling: inside the iscsi_segment, we
  79  * remember an index into the scatterlist, and set data/size
  80  * to the current scatterlist entry. For highmem pages, we
  81  * kmap as needed.
  82  *
  83  * Note that the page is unmapped when we return from
  84  * TCP's data_ready handler, so we may end up mapping and
  85  * unmapping the same page repeatedly. The whole reason
  86  * for this is that we shouldn't keep the page mapped
  87  * outside the softirq.
  88  */
  89
  90 /**
  91  * iscsi_tcp_segment_init_sg - init indicated scatterlist entry
  92  * @segment: the buffer object
  93  * @sg: scatterlist
  94  * @offset: byte offset into that sg entry
  95  *
  96  * This function sets up the segment so that subsequent
  97  * data is copied to the indicated sg entry, at the given
  98  * offset.
  99  */
 100 static inline void
 101 iscsi_tcp_segment_init_sg(struct iscsi_segment *segment,
 102                           struct scatterlist *sg, unsigned int offset)
 103 {
 104         segment->sg = sg;
 105         segment->sg_offset = offset;
 106         segment->size = min(sg->length - offset,
 107                             segment->total_size - segment->total_copied);
 108         segment->data = NULL;
 109 }
 110
 111 /**
 112  * iscsi_tcp_segment_map - map the current S/G page
 113  * @segment: iscsi_segment
 114  * @recv: 1 if called from recv path
 115  *
 116  * We only need to possibly kmap data if scatter lists are being used,
 117  * because the iscsi passthrough and internal IO paths will never use high
 118  * mem pages.
 119  */
 120 static inline void
 121 iscsi_tcp_segment_map(struct iscsi_segment *segment, int recv)
 122 {
 123         struct scatterlist *sg;
 124
 125         if (segment->data != NULL || !segment->sg)
 126                 return;
 127
 128         sg = segment->sg;
 129         BUG_ON(segment->sg_mapped);
 130         BUG_ON(sg->length == 0);
 131
 132         /*
 133          * If the page count is greater than one it is ok to send
 134          * to the network layer's zero copy send path. If not we
 135          * have to go the slow sendmsg path. We always map for the
 136          * recv path.
 137          */
 138         if (page_count(sg_page(sg)) >= 1 && !recv)
 139                 return;
 140
 141         debug_tcp("iscsi_tcp_segment_map %s %p\n", recv ? "recv" : "xmit",
 142                   segment);
 143         segment->sg_mapped = kmap_atomic(sg_page(sg), KM_SOFTIRQ0);
 144         segment->data = segment->sg_mapped + sg->offset + segment->sg_offset;
 145 }
 146
 147 static inline void
 148 iscsi_tcp_segment_unmap(struct iscsi_segment *segment)
 149 {
 150         debug_tcp("iscsi_tcp_segment_unmap %p\n", segment);
 151
 152         if (segment->sg_mapped) {
 153                 debug_tcp("iscsi_tcp_segment_unmap valid\n");
 154                 kunmap_atomic(segment->sg_mapped, KM_SOFTIRQ0);
 155                 segment->sg_mapped = NULL;
 156                 segment->data = NULL;
 157         }
 158 }
 159
 160 /*
 161  * Splice the digest buffer into the buffer
 162  */
 163 static inline void
 164 iscsi_tcp_segment_splice_digest(struct iscsi_segment *segment, void *digest)
 165 {
 166         segment->data = digest;
 167         segment->digest_len = ISCSI_DIGEST_SIZE;
 168         segment->total_size += ISCSI_DIGEST_SIZE;
 169         segment->size = ISCSI_DIGEST_SIZE;
 170         segment->copied = 0;
 171         segment->sg = NULL;
 172         segment->hash = NULL;
 173 }
 174
 175 /**
 176  * iscsi_tcp_segment_done - check whether the segment is complete
 177  * @segment: iscsi segment to check
 178  * @recv: set to one of this is called from the recv path
 179  * @copied: number of bytes copied
 180  *
 181  * Check if we're done receiving this segment. If the receive
 182  * buffer is full but we expect more data, move on to the
 183  * next entry in the scatterlist.
 184  *
 185  * If the amount of data we received isn't a multiple of 4,
 186  * we will transparently receive the pad bytes, too.
 187  *
 188  * This function must be re-entrant.
 189  */
 190 static inline int
 191 iscsi_tcp_segment_done(struct iscsi_segment *segment, int recv, unsigned copied)
 192 {
 193         static unsigned char padbuf[ISCSI_PAD_LEN];
 194         struct scatterlist sg;
 195         unsigned int pad;
 196
 197         debug_tcp("copied %u %u size %u %s\n", segment->copied, copied,
 198                   segment->size, recv ? "recv" : "xmit");
 199         if (segment->hash && copied) {
 200                 /*
 201                  * If a segment is kmapd we must unmap it before sending
 202                  * to the crypto layer since that will try to kmap it again.
 203                  */
 204                 iscsi_tcp_segment_unmap(segment);
 205
 206                 if (!segment->data) {
 207                         sg_init_table(&sg, 1);
 208                         sg_set_page(&sg, sg_page(segment->sg), copied,
 209                                     segment->copied + segment->sg_offset +
 210                                                         segment->sg->offset);
 211                 } else
 212                         sg_init_one(&sg, segment->data + segment->copied,
 213                                     copied);
 214                 crypto_hash_update(segment->hash, &sg, copied);
 215         }
 216
 217         segment->copied += copied;
 218         if (segment->copied < segment->size) {
 219                 iscsi_tcp_segment_map(segment, recv);
 220                 return 0;
 221         }
 222
 223         segment->total_copied += segment->copied;
 224         segment->copied = 0;
 225         segment->size = 0;
 226
 227         /* Unmap the current scatterlist page, if there is one. */
 228         iscsi_tcp_segment_unmap(segment);
 229
 230         /* Do we have more scatterlist entries? */
 231         debug_tcp("total copied %u total size %u\n", segment->total_copied,
 232                    segment->total_size);
 233         if (segment->total_copied < segment->total_size) {
 234                 /* Proceed to the next entry in the scatterlist. */
 235                 iscsi_tcp_segment_init_sg(segment, sg_next(segment->sg),
 236                                           0);
 237                 iscsi_tcp_segment_map(segment, recv);
 238                 BUG_ON(segment->size == 0);
 239                 return 0;
 240         }
 241
 242         /* Do we need to handle padding? */
 243         pad = iscsi_padding(segment->total_copied);
 244         if (pad != 0) {
 245                 debug_tcp("consume %d pad bytes\n", pad);
 246                 segment->total_size += pad;
 247                 segment->size = pad;
 248                 segment->data = padbuf;
 249                 return 0;
 250         }
 251
 252         /*
 253          * Set us up for transferring the data digest. hdr digest
 254          * is completely handled in hdr done function.
 255          */
 256         if (segment->hash) {
 257                 crypto_hash_final(segment->hash, segment->digest);
 258                 iscsi_tcp_segment_splice_digest(segment,
 259                                  recv ? segment->recv_digest : segment->digest);
 260                 return 0;
 261         }
 262
 263         return 1;
 264 }
 265
 266 /**
 267  * iscsi_tcp_xmit_segment - transmit segment
 268  * @tcp_conn: the iSCSI TCP connection
 269  * @segment: the buffer to transmnit
 270  *
 271  * This function transmits as much of the buffer as
 272  * the network layer will accept, and returns the number of
 273  * bytes transmitted.
 274  *
 275  * If CRC hashing is enabled, the function will compute the
 276  * hash as it goes. When the entire segment has been transmitted,
 277  * it will retrieve the hash value and send it as well.
 278  */
 279 static int
 280 iscsi_tcp_xmit_segment(struct iscsi_tcp_conn *tcp_conn,
 281                        struct iscsi_segment *segment)
 282 {
 283         struct socket *sk = tcp_conn->sock;
 284         unsigned int copied = 0;
 285         int r = 0;
 286
 287         while (!iscsi_tcp_segment_done(segment, 0, r)) {
 288                 struct scatterlist *sg;
 289                 unsigned int offset, copy;
 290                 int flags = 0;
 291
 292                 r = 0;
 293                 offset = segment->copied;
 294                 copy = segment->size - offset;
 295
 296                 if (segment->total_copied + segment->size < segment->total_size)
 297                         flags |= MSG_MORE;
 298
 299                 /* Use sendpage if we can; else fall back to sendmsg */
 300                 if (!segment->data) {
 301                         sg = segment->sg;
 302                         offset += segment->sg_offset + sg->offset;
 303                         r = tcp_conn->sendpage(sk, sg_page(sg), offset, copy,
 304                                                flags);
 305                 } else {
 306                         struct msghdr msg = { .msg_flags = flags };
 307                         struct kvec iov = {
 308                                 .iov_base = segment->data + offset,
 309                                 .iov_len = copy
 310                         };
 311
 312                         r = kernel_sendmsg(sk, &msg, &iov, 1, copy);
 313                 }
 314
 315                 if (r < 0) {
 316                         iscsi_tcp_segment_unmap(segment);
 317                         if (copied || r == -EAGAIN)
 318                                 break;
 319                         return r;
 320                 }
 321                 copied += r;
 322         }
 323         return copied;
 324 }
 325
 326 /**
 327  * iscsi_tcp_segment_recv - copy data to segment
 328  * @tcp_conn: the iSCSI TCP connection
 329  * @segment: the buffer to copy to
 330  * @ptr: data pointer
 331  * @len: amount of data available
 332  *
 333  * This function copies up to @len bytes to the
 334  * given buffer, and returns the number of bytes
 335  * consumed, which can actually be less than @len.
 336  *
 337  * If hash digest is enabled, the function will update the
 338  * hash while copying.
 339  * Combining these two operations doesn't buy us a lot (yet),
 340  * but in the future we could implement combined copy+crc,
 341  * just way we do for network layer checksums.
 342  */
 343 static int
 344 iscsi_tcp_segment_recv(struct iscsi_tcp_conn *tcp_conn,
 345                        struct iscsi_segment *segment, const void *ptr,
 346                        unsigned int len)
 347 {
 348         unsigned int copy = 0, copied = 0;
 349
 350         while (!iscsi_tcp_segment_done(segment, 1, copy)) {
 351                 if (copied == len) {
 352                         debug_tcp("iscsi_tcp_segment_recv copied %d bytes\n",
 353                                   len);
 354                         break;
 355                 }
 356
 357                 copy = min(len - copied, segment->size - segment->copied);
 358                 debug_tcp("iscsi_tcp_segment_recv copying %d\n", copy);
 359                 memcpy(segment->data + segment->copied, ptr + copied, copy);
 360                 copied += copy;
 361         }
 362         return copied;
 363 }
 364
 365 static inline void
 366 iscsi_tcp_dgst_header(struct hash_desc *hash, const void *hdr, size_t hdrlen,
 367                       unsigned char digest[ISCSI_DIGEST_SIZE])
 368 {
 369         struct scatterlist sg;
 370
 371         sg_init_one(&sg, hdr, hdrlen);
 372         crypto_hash_digest(hash, &sg, hdrlen, digest);
 373 }
 374
 375 static inline int
 376 iscsi_tcp_dgst_verify(struct iscsi_tcp_conn *tcp_conn,
 377                       struct iscsi_segment *segment)
 378 {
 379         if (!segment->digest_len)
 380                 return 1;
 381
 382         if (memcmp(segment->recv_digest, segment->digest,
 383                    segment->digest_len)) {
 384                 debug_scsi("digest mismatch\n");
 385                 return 0;
 386         }
 387
 388         return 1;
 389 }
 390
 391 /*
 392  * Helper function to set up segment buffer
 393  */
 394 static inline void
 395 __iscsi_segment_init(struct iscsi_segment *segment, size_t size,
 396                      iscsi_segment_done_fn_t *done, struct hash_desc *hash)
 397 {
 398         memset(segment, 0, sizeof(*segment));
 399         segment->total_size = size;
 400         segment->done = done;
 401
 402         if (hash) {
 403                 segment->hash = hash;
 404                 crypto_hash_init(hash);
 405         }
 406 }
 407
 408 static inline void
 409 iscsi_segment_init_linear(struct iscsi_segment *segment, void *data,
 410                           size_t size, iscsi_segment_done_fn_t *done,
 411                           struct hash_desc *hash)
 412 {
 413         __iscsi_segment_init(segment, size, done, hash);
 414         segment->data = data;
 415         segment->size = size;
 416 }
 417
 418 static inline int
 419 iscsi_segment_seek_sg(struct iscsi_segment *segment,
 420                       struct scatterlist *sg_list, unsigned int sg_count,
 421                       unsigned int offset, size_t size,
 422                       iscsi_segment_done_fn_t *done, struct hash_desc *hash)
 423 {
 424         struct scatterlist *sg;
 425         unsigned int i;
 426
 427         debug_scsi("iscsi_segment_seek_sg offset %u size %llu\n",
 428                   offset, size);
 429         __iscsi_segment_init(segment, size, done, hash);
 430         for_each_sg(sg_list, sg, sg_count, i) {
 431                 debug_scsi("sg %d, len %u offset %u\n", i, sg->length,
 432                            sg->offset);
 433                 if (offset < sg->length) {
 434                         iscsi_tcp_segment_init_sg(segment, sg, offset);
 435                         return 0;
 436                 }
 437                 offset -= sg->length;
 438         }
 439
 440         return ISCSI_ERR_DATA_OFFSET;
 441 }
 442
 443 /**
 444  * iscsi_tcp_hdr_recv_prep - prep segment for hdr reception
 445  * @tcp_conn: iscsi connection to prep for
 446  *
 447  * This function always passes NULL for the hash argument, because when this
 448  * function is called we do not yet know the final size of the header and want
 449  * to delay the digest processing until we know that.
 450  */
 451 static void
 452 iscsi_tcp_hdr_recv_prep(struct iscsi_tcp_conn *tcp_conn)
 453 {
 454         debug_tcp("iscsi_tcp_hdr_recv_prep(%p%s)\n", tcp_conn,
 455                   tcp_conn->iscsi_conn->hdrdgst_en ? ", digest enabled" : "");
 456         iscsi_segment_init_linear(&tcp_conn->in.segment,
 457                                 tcp_conn->in.hdr_buf, sizeof(struct iscsi_hdr),
 458                                 iscsi_tcp_hdr_recv_done, NULL);
 459 }
 460
 461 /*
 462  * Handle incoming reply to any other type of command
 463  */
 464 static int
 465 iscsi_tcp_data_recv_done(struct iscsi_tcp_conn *tcp_conn,
 466                          struct iscsi_segment *segment)
 467 {
 468         struct iscsi_conn *conn = tcp_conn->iscsi_conn;
 469         int rc = 0;
 470
 471         if (!iscsi_tcp_dgst_verify(tcp_conn, segment))
 472                 return ISCSI_ERR_DATA_DGST;
 473
 474         rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr,
 475                         conn->data, tcp_conn->in.datalen);
 476         if (rc)
 477                 return rc;
 478
 479         iscsi_tcp_hdr_recv_prep(tcp_conn);
 480         return 0;
 481 }
 482
 483 static void
 484 iscsi_tcp_data_recv_prep(struct iscsi_tcp_conn *tcp_conn)
 485 {
 486         struct iscsi_conn *conn = tcp_conn->iscsi_conn;
 487         struct hash_desc *rx_hash = NULL;
 488
 489         if (conn->datadgst_en &
 490             !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD))
 491                 rx_hash = &tcp_conn->rx_hash;
 492
 493         iscsi_segment_init_linear(&tcp_conn->in.segment,
 494                                 conn->data, tcp_conn->in.datalen,
 495                                 iscsi_tcp_data_recv_done, rx_hash);
 496 }
 497
 498 /*
 499  * must be called with session lock
 500  */
 501 static void
 502 iscsi_tcp_cleanup_task(struct iscsi_conn *conn, struct iscsi_task *task)
 503 {
 504         struct iscsi_tcp_task *tcp_task = task->dd_data;
 505         struct iscsi_r2t_info *r2t;
 506
 507         /* nothing to do for mgmt tasks */
 508         if (!task->sc)
 509                 return;
 510
 511         /* flush task's r2t queues */
 512         while (__kfifo_get(tcp_task->r2tqueue, (void*)&r2t, sizeof(void*))) {
 513                 __kfifo_put(tcp_task->r2tpool.queue, (void*)&r2t,
 514                             sizeof(void*));
 515                 debug_scsi("iscsi_tcp_cleanup_task pending r2t dropped\n");
 516         }
 517
 518         r2t = tcp_task->r2t;
 519         if (r2t != NULL) {
 520                 __kfifo_put(tcp_task->r2tpool.queue, (void*)&r2t,
 521                             sizeof(void*));
 522                 tcp_task->r2t = NULL;
 523         }
 524 }
 525
 526 /**
 527  * iscsi_data_in - SCSI Data-In Response processing
 528  * @conn: iscsi connection
 529  * @task: scsi command task
 530  **/
 531 static int
 532 iscsi_data_in(struct iscsi_conn *conn, struct iscsi_task *task)
 533 {
 534         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 535         struct iscsi_tcp_task *tcp_task = task->dd_data;
 536         struct iscsi_data_rsp *rhdr = (struct iscsi_data_rsp *)tcp_conn->in.hdr;
 537         int datasn = be32_to_cpu(rhdr->datasn);
 538         unsigned total_in_length = scsi_in(task->sc)->length;
 539
 540         iscsi_update_cmdsn(conn->session, (struct iscsi_nopin*)rhdr);
 541         if (tcp_conn->in.datalen == 0)
 542                 return 0;
 543
 544         if (tcp_task->exp_datasn != datasn) {
 545                 debug_tcp("%s: task->exp_datasn(%d) != rhdr->datasn(%d)\n",
 546                           __func__, tcp_task->exp_datasn, datasn);
 547                 return ISCSI_ERR_DATASN;
 548         }
 549
 550         tcp_task->exp_datasn++;
 551
 552         tcp_task->data_offset = be32_to_cpu(rhdr->offset);
 553         if (tcp_task->data_offset + tcp_conn->in.datalen > total_in_length) {
 554                 debug_tcp("%s: data_offset(%d) + data_len(%d) > total_length_in(%d)\n",
 555                           __func__, tcp_task->data_offset,
 556                           tcp_conn->in.datalen, total_in_length);
 557                 return ISCSI_ERR_DATA_OFFSET;
 558         }
 559
 560         conn->datain_pdus_cnt++;
 561         return 0;
 562 }
 563
 564 /**
 565  * iscsi_solicit_data_init - initialize first Data-Out
 566  * @conn: iscsi connection
 567  * @task: scsi command task
 568  * @r2t: R2T info
 569  *
 570  * Notes:
 571  *      Initialize first Data-Out within this R2T sequence and finds
 572  *      proper data_offset within this SCSI command.
 573  *
 574  *      This function is called with connection lock taken.
 575  **/
 576 static void
 577 iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_task *task,
 578                         struct iscsi_r2t_info *r2t)
 579 {
 580         struct iscsi_data *hdr;
 581
 582         hdr = &r2t->dtask.hdr;
 583         memset(hdr, 0, sizeof(struct iscsi_data));
 584         hdr->ttt = r2t->ttt;
 585         hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
 586         r2t->solicit_datasn++;
 587         hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
 588         memcpy(hdr->lun, task->hdr->lun, sizeof(hdr->lun));
 589         hdr->itt = task->hdr->itt;
 590         hdr->exp_statsn = r2t->exp_statsn;
 591         hdr->offset = cpu_to_be32(r2t->data_offset);
 592         if (r2t->data_length > conn->max_xmit_dlength) {
 593                 hton24(hdr->dlength, conn->max_xmit_dlength);
 594                 r2t->data_count = conn->max_xmit_dlength;
 595                 hdr->flags = 0;
 596         } else {
 597                 hton24(hdr->dlength, r2t->data_length);
 598                 r2t->data_count = r2t->data_length;
 599                 hdr->flags = ISCSI_FLAG_CMD_FINAL;
 600         }
 601         conn->dataout_pdus_cnt++;
 602
 603         r2t->sent = 0;
 604 }
 605
 606 /**
 607  * iscsi_r2t_rsp - iSCSI R2T Response processing
 608  * @conn: iscsi connection
 609  * @task: scsi command task
 610  **/
 611 static int
 612 iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_task *task)
 613 {
 614         struct iscsi_r2t_info *r2t;
 615         struct iscsi_session *session = conn->session;
 616         struct iscsi_tcp_task *tcp_task = task->dd_data;
 617         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 618         struct iscsi_r2t_rsp *rhdr = (struct iscsi_r2t_rsp *)tcp_conn->in.hdr;
 619         int r2tsn = be32_to_cpu(rhdr->r2tsn);
 620         int rc;
 621
 622         if (tcp_conn->in.datalen) {
 623                 iscsi_conn_printk(KERN_ERR, conn,
 624                                   "invalid R2t with datalen %d\n",
 625                                   tcp_conn->in.datalen);
 626                 return ISCSI_ERR_DATALEN;
 627         }
 628
 629         if (tcp_task->exp_datasn != r2tsn){
 630                 debug_tcp("%s: task->exp_datasn(%d) != rhdr->r2tsn(%d)\n",
 631                           __func__, tcp_task->exp_datasn, r2tsn);
 632                 return ISCSI_ERR_R2TSN;
 633         }
 634
 635         /* fill-in new R2T associated with the task */
 636         iscsi_update_cmdsn(session, (struct iscsi_nopin*)rhdr);
 637
 638         if (!task->sc || session->state != ISCSI_STATE_LOGGED_IN) {
 639                 iscsi_conn_printk(KERN_INFO, conn,
 640                                   "dropping R2T itt %d in recovery.\n",
 641                                   task->itt);
 642                 return 0;
 643         }
 644
 645         rc = __kfifo_get(tcp_task->r2tpool.queue, (void*)&r2t, sizeof(void*));
 646         BUG_ON(!rc);
 647
 648         r2t->exp_statsn = rhdr->statsn;
 649         r2t->data_length = be32_to_cpu(rhdr->data_length);
 650         if (r2t->data_length == 0) {
 651                 iscsi_conn_printk(KERN_ERR, conn,
 652                                   "invalid R2T with zero data len\n");
 653                 __kfifo_put(tcp_task->r2tpool.queue, (void*)&r2t,
 654                             sizeof(void*));
 655                 return ISCSI_ERR_DATALEN;
 656         }
 657
 658         if (r2t->data_length > session->max_burst)
 659                 debug_scsi("invalid R2T with data len %u and max burst %u."
 660                            "Attempting to execute request.\n",
 661                             r2t->data_length, session->max_burst);
 662
 663         r2t->data_offset = be32_to_cpu(rhdr->data_offset);
 664         if (r2t->data_offset + r2t->data_length > scsi_out(task->sc)->length) {
 665                 iscsi_conn_printk(KERN_ERR, conn,
 666                                   "invalid R2T with data len %u at offset %u "
 667                                   "and total length %d\n", r2t->data_length,
 668                                   r2t->data_offset, scsi_out(task->sc)->length);
 669                 __kfifo_put(tcp_task->r2tpool.queue, (void*)&r2t,
 670                             sizeof(void*));
 671                 return ISCSI_ERR_DATALEN;
 672         }
 673
 674         r2t->ttt = rhdr->ttt; /* no flip */
 675         r2t->solicit_datasn = 0;
 676
 677         iscsi_solicit_data_init(conn, task, r2t);
 678
 679         tcp_task->exp_datasn = r2tsn + 1;
 680         __kfifo_put(tcp_task->r2tqueue, (void*)&r2t, sizeof(void*));
 681         conn->r2t_pdus_cnt++;
 682
 683         iscsi_requeue_task(task);
 684         return 0;
 685 }
 686
 687 /*
 688  * Handle incoming reply to DataIn command
 689  */
 690 static int
 691 iscsi_tcp_process_data_in(struct iscsi_tcp_conn *tcp_conn,
 692                           struct iscsi_segment *segment)
 693 {
 694         struct iscsi_conn *conn = tcp_conn->iscsi_conn;
 695         struct iscsi_hdr *hdr = tcp_conn->in.hdr;
 696         int rc;
 697
 698         if (!iscsi_tcp_dgst_verify(tcp_conn, segment))
 699                 return ISCSI_ERR_DATA_DGST;
 700
 701         /* check for non-exceptional status */
 702         if (hdr->flags & ISCSI_FLAG_DATA_STATUS) {
 703                 rc = iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0);
 704                 if (rc)
 705                         return rc;
 706         }
 707
 708         iscsi_tcp_hdr_recv_prep(tcp_conn);
 709         return 0;
 710 }
 711
 712 /**
 713  * iscsi_tcp_hdr_dissect - process PDU header
 714  * @conn: iSCSI connection
 715  * @hdr: PDU header
 716  *
 717  * This function analyzes the header of the PDU received,
 718  * and performs several sanity checks. If the PDU is accompanied
 719  * by data, the receive buffer is set up to copy the incoming data
 720  * to the correct location.
 721  */
 722 static int
 723 iscsi_tcp_hdr_dissect(struct iscsi_conn *conn, struct iscsi_hdr *hdr)
 724 {
 725         int rc = 0, opcode, ahslen;
 726         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 727         struct iscsi_task *task;
 728
 729         /* verify PDU length */
 730         tcp_conn->in.datalen = ntoh24(hdr->dlength);
 731         if (tcp_conn->in.datalen > conn->max_recv_dlength) {
 732                 iscsi_conn_printk(KERN_ERR, conn,
 733                                   "iscsi_tcp: datalen %d > %d\n",
 734                                   tcp_conn->in.datalen, conn->max_recv_dlength);
 735                 return ISCSI_ERR_DATALEN;
 736         }
 737
 738         /* Additional header segments. So far, we don't
 739          * process additional headers.
 740          */
 741         ahslen = hdr->hlength << 2;
 742
 743         opcode = hdr->opcode & ISCSI_OPCODE_MASK;
 744         /* verify itt (itt encoding: age+cid+itt) */
 745         rc = iscsi_verify_itt(conn, hdr->itt);
 746         if (rc)
 747                 return rc;
 748
 749         debug_tcp("opcode 0x%x ahslen %d datalen %d\n",
 750                   opcode, ahslen, tcp_conn->in.datalen);
 751
 752         switch(opcode) {
 753         case ISCSI_OP_SCSI_DATA_IN:
 754                 spin_lock(&conn->session->lock);
 755                 task = iscsi_itt_to_ctask(conn, hdr->itt);
 756                 if (!task)
 757                         rc = ISCSI_ERR_BAD_ITT;
 758                 else
 759                         rc = iscsi_data_in(conn, task);
 760                 if (rc) {
 761                         spin_unlock(&conn->session->lock);
 762                         break;
 763                 }
 764
 765                 if (tcp_conn->in.datalen) {
 766                         struct iscsi_tcp_task *tcp_task = task->dd_data;
 767                         struct hash_desc *rx_hash = NULL;
 768                         struct scsi_data_buffer *sdb = scsi_in(task->sc);
 769
 770                         /*
 771                          * Setup copy of Data-In into the Scsi_Cmnd
 772                          * Scatterlist case:
 773                          * We set up the iscsi_segment to point to the next
 774                          * scatterlist entry to copy to. As we go along,
 775                          * we move on to the next scatterlist entry and
 776                          * update the digest per-entry.
 777                          */
 778                         if (conn->datadgst_en &&
 779                             !(conn->session->tt->caps & CAP_DIGEST_OFFLOAD))
 780                                 rx_hash = &tcp_conn->rx_hash;
 781
 782                         debug_tcp("iscsi_tcp_begin_data_in(%p, offset=%d, "
 783                                   "datalen=%d)\n", tcp_conn,
 784                                   tcp_task->data_offset,
 785                                   tcp_conn->in.datalen);
 786                         rc = iscsi_segment_seek_sg(&tcp_conn->in.segment,
 787                                                    sdb->table.sgl,
 788                                                    sdb->table.nents,
 789                                                    tcp_task->data_offset,
 790                                                    tcp_conn->in.datalen,
 791                                                    iscsi_tcp_process_data_in,
 792                                                    rx_hash);
 793                         spin_unlock(&conn->session->lock);
 794                         return rc;
 795                 }
 796                 rc = __iscsi_complete_pdu(conn, hdr, NULL, 0);
 797                 spin_unlock(&conn->session->lock);
 798                 break;
 799         case ISCSI_OP_SCSI_CMD_RSP:
 800                 if (tcp_conn->in.datalen) {
 801                         iscsi_tcp_data_recv_prep(tcp_conn);
 802                         return 0;
 803                 }
 804                 rc = iscsi_complete_pdu(conn, hdr, NULL, 0);
 805                 break;
 806         case ISCSI_OP_R2T:
 807                 spin_lock(&conn->session->lock);
 808                 task = iscsi_itt_to_ctask(conn, hdr->itt);
 809                 if (!task)
 810                         rc = ISCSI_ERR_BAD_ITT;
 811                 else if (ahslen)
 812                         rc = ISCSI_ERR_AHSLEN;
 813                 else if (task->sc->sc_data_direction == DMA_TO_DEVICE)
 814                         rc = iscsi_r2t_rsp(conn, task);
 815                 else
 816                         rc = ISCSI_ERR_PROTO;
 817                 spin_unlock(&conn->session->lock);
 818                 break;
 819         case ISCSI_OP_LOGIN_RSP:
 820         case ISCSI_OP_TEXT_RSP:
 821         case ISCSI_OP_REJECT:
 822         case ISCSI_OP_ASYNC_EVENT:
 823                 /*
 824                  * It is possible that we could get a PDU with a buffer larger
 825                  * than 8K, but there are no targets that currently do this.
 826                  * For now we fail until we find a vendor that needs it
 827                  */
 828                 if (ISCSI_DEF_MAX_RECV_SEG_LEN < tcp_conn->in.datalen) {
 829                         iscsi_conn_printk(KERN_ERR, conn,
 830                                           "iscsi_tcp: received buffer of "
 831                                           "len %u but conn buffer is only %u "
 832                                           "(opcode %0x)\n",
 833                                           tcp_conn->in.datalen,
 834                                           ISCSI_DEF_MAX_RECV_SEG_LEN, opcode);
 835                         rc = ISCSI_ERR_PROTO;
 836                         break;
 837                 }
 838
 839                 /* If there's data coming in with the response,
 840                  * receive it to the connection's buffer.
 841                  */
 842                 if (tcp_conn->in.datalen) {
 843                         iscsi_tcp_data_recv_prep(tcp_conn);
 844                         return 0;
 845                 }
 846         /* fall through */
 847         case ISCSI_OP_LOGOUT_RSP:
 848         case ISCSI_OP_NOOP_IN:
 849         case ISCSI_OP_SCSI_TMFUNC_RSP:
 850                 rc = iscsi_complete_pdu(conn, hdr, NULL, 0);
 851                 break;
 852         default:
 853                 rc = ISCSI_ERR_BAD_OPCODE;
 854                 break;
 855         }
 856
 857         if (rc == 0) {
 858                 /* Anything that comes with data should have
 859                  * been handled above. */
 860                 if (tcp_conn->in.datalen)
 861                         return ISCSI_ERR_PROTO;
 862                 iscsi_tcp_hdr_recv_prep(tcp_conn);
 863         }
 864
 865         return rc;
 866 }
 867
 868 /**
 869  * iscsi_tcp_hdr_recv_done - process PDU header
 870  *
 871  * This is the callback invoked when the PDU header has
 872  * been received. If the header is followed by additional
 873  * header segments, we go back for more data.
 874  */
 875 static int
 876 iscsi_tcp_hdr_recv_done(struct iscsi_tcp_conn *tcp_conn,
 877                         struct iscsi_segment *segment)
 878 {
 879         struct iscsi_conn *conn = tcp_conn->iscsi_conn;
 880         struct iscsi_hdr *hdr;
 881
 882         /* Check if there are additional header segments
 883          * *prior* to computing the digest, because we
 884          * may need to go back to the caller for more.
 885          */
 886         hdr = (struct iscsi_hdr *) tcp_conn->in.hdr_buf;
 887         if (segment->copied == sizeof(struct iscsi_hdr) && hdr->hlength) {
 888                 /* Bump the header length - the caller will
 889                  * just loop around and get the AHS for us, and
 890                  * call again. */
 891                 unsigned int ahslen = hdr->hlength << 2;
 892
 893                 /* Make sure we don't overflow */
 894                 if (sizeof(*hdr) + ahslen > sizeof(tcp_conn->in.hdr_buf))
 895                         return ISCSI_ERR_AHSLEN;
 896
 897                 segment->total_size += ahslen;
 898                 segment->size += ahslen;
 899                 return 0;
 900         }
 901
 902         /* We're done processing the header. See if we're doing
 903          * header digests; if so, set up the recv_digest buffer
 904          * and go back for more. */
 905         if (conn->hdrdgst_en) {
 906                 if (segment->digest_len == 0) {
 907                         /*
 908                          * Even if we offload the digest processing we
 909                          * splice it in so we can increment the skb/segment
 910                          * counters in preparation for the data segment.
 911                          */
 912                         iscsi_tcp_segment_splice_digest(segment,
 913                                                         segment->recv_digest);
 914                         return 0;
 915                 }
 916
 917                 if (!(conn->session->tt->caps & CAP_DIGEST_OFFLOAD)) {
 918                         iscsi_tcp_dgst_header(&tcp_conn->rx_hash, hdr,
 919                                 segment->total_copied - ISCSI_DIGEST_SIZE,
 920                                 segment->digest);
 921
 922                         if (!iscsi_tcp_dgst_verify(tcp_conn, segment))
 923                                 return ISCSI_ERR_HDR_DGST;
 924                 }
 925         }
 926
 927         tcp_conn->in.hdr = hdr;
 928         return iscsi_tcp_hdr_dissect(conn, hdr);
 929 }
 930
 931 inline int iscsi_tcp_recv_segment_is_hdr(struct iscsi_tcp_conn *tcp_conn)
 932 {
 933         return tcp_conn->in.segment.done == iscsi_tcp_hdr_recv_done;
 934 }
 935
 936 enum {
 937         ISCSI_TCP_SEGMENT_DONE,         /* curr seg has been processed */
 938         ISCSI_TCP_SKB_DONE,             /* skb is out of data */
 939         ISCSI_TCP_CONN_ERR,             /* iscsi layer has fired a conn err */
 940         ISCSI_TCP_SUSPENDED,            /* conn is suspended */
 941 };
 942
 943 /**
 944  * iscsi_tcp_recv_skb - Process skb
 945  * @conn: iscsi connection
 946  * @skb: network buffer with header and/or data segment
 947  * @offset: offset in skb
 948  * @offload: bool indicating if transfer was offloaded
 949  */
 950 int iscsi_tcp_recv_skb(struct iscsi_conn *conn, struct sk_buff *skb,
 951                        unsigned int offset, bool offloaded, int *status)
 952 {
 953         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 954         struct iscsi_segment *segment = &tcp_conn->in.segment;
 955         struct skb_seq_state seq;
 956         unsigned int consumed = 0;
 957         int rc = 0;
 958
 959         debug_tcp("in %d bytes\n", skb->len - offset);
 960
 961         if (unlikely(conn->suspend_rx)) {
 962                 debug_tcp("conn %d Rx suspended!\n", conn->id);
 963                 *status = ISCSI_TCP_SUSPENDED;
 964                 return 0;
 965         }
 966
 967         if (offloaded) {
 968                 segment->total_copied = segment->total_size;
 969                 goto segment_done;
 970         }
 971
 972         skb_prepare_seq_read(skb, offset, skb->len, &seq);
 973         while (1) {
 974                 unsigned int avail;
 975                 const u8 *ptr;
 976
 977                 avail = skb_seq_read(consumed, &ptr, &seq);
 978                 if (avail == 0) {
 979                         debug_tcp("no more data avail. Consumed %d\n",
 980                                   consumed);
 981                         *status = ISCSI_TCP_SKB_DONE;
 982                         skb_abort_seq_read(&seq);
 983                         goto skb_done;
 984                 }
 985                 BUG_ON(segment->copied >= segment->size);
 986
 987                 debug_tcp("skb %p ptr=%p avail=%u\n", skb, ptr, avail);
 988                 rc = iscsi_tcp_segment_recv(tcp_conn, segment, ptr, avail);
 989                 BUG_ON(rc == 0);
 990                 consumed += rc;
 991
 992                 if (segment->total_copied >= segment->total_size) {
 993                         skb_abort_seq_read(&seq);
 994                         goto segment_done;
 995                 }
 996         }
 997
 998 segment_done:
 999         *status = ISCSI_TCP_SEGMENT_DONE;
1000         debug_tcp("segment done\n");
1001         rc = segment->done(tcp_conn, segment);
1002         if (rc != 0) {
1003                 *status = ISCSI_TCP_CONN_ERR;
1004                 debug_tcp("Error receiving PDU, errno=%d\n", rc);
1005                 iscsi_conn_failure(conn, rc);
1006                 return 0;
1007         }
1008         /* The done() functions sets up the next segment. */
1009
1010 skb_done:
1011         conn->rxdata_octets += consumed;
1012         return consumed;
1013 }
1014 EXPORT_SYMBOL_GPL(iscsi_tcp_recv_skb);
1015
1016 /**
1017  * iscsi_tcp_recv - TCP receive in sendfile fashion
1018  * @rd_desc: read descriptor
1019  * @skb: socket buffer
1020  * @offset: offset in skb
1021  * @len: skb->len - offset
1022  **/
1023 static int
1024 iscsi_tcp_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
1025                unsigned int offset, size_t len)
1026 {
1027         struct iscsi_conn *conn = rd_desc->arg.data;
1028         unsigned int consumed, total_consumed = 0;
1029         int status;
1030
1031         debug_tcp("in %d bytes\n", skb->len - offset);
1032
1033         do {
1034                 status = 0;
1035                 consumed = iscsi_tcp_recv_skb(conn, skb, offset, 0, &status);
1036                 offset += consumed;
1037                 total_consumed += consumed;
1038         } while (consumed != 0 && status != ISCSI_TCP_SKB_DONE);
1039
1040         debug_tcp("read %d bytes status %d\n", skb->len - offset, status);
1041         return total_consumed;
1042 }
1043
1044 static void
1045 iscsi_tcp_data_ready(struct sock *sk, int flag)
1046 {
1047         struct iscsi_conn *conn = sk->sk_user_data;
1048         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1049         read_descriptor_t rd_desc;
1050
1051         read_lock(&sk->sk_callback_lock);
1052
1053         /*
1054          * Use rd_desc to pass 'conn' to iscsi_tcp_recv.
1055          * We set count to 1 because we want the network layer to
1056          * hand us all the skbs that are available. iscsi_tcp_recv
1057          * handled pdus that cross buffers or pdus that still need data.
1058          */
1059         rd_desc.arg.data = conn;
1060         rd_desc.count = 1;
1061         tcp_read_sock(sk, &rd_desc, iscsi_tcp_recv);
1062
1063         read_unlock(&sk->sk_callback_lock);
1064
1065         /* If we had to (atomically) map a highmem page,
1066          * unmap it now. */
1067         iscsi_tcp_segment_unmap(&tcp_conn->in.segment);
1068 }
1069
1070 static void
1071 iscsi_tcp_state_change(struct sock *sk)
1072 {
1073         struct iscsi_tcp_conn *tcp_conn;
1074         struct iscsi_conn *conn;
1075         struct iscsi_session *session;
1076         void (*old_state_change)(struct sock *);
1077
1078         read_lock(&sk->sk_callback_lock);
1079
1080         conn = (struct iscsi_conn*)sk->sk_user_data;
1081         session = conn->session;
1082
1083         if ((sk->sk_state == TCP_CLOSE_WAIT ||
1084              sk->sk_state == TCP_CLOSE) &&
1085             !atomic_read(&sk->sk_rmem_alloc)) {
1086                 debug_tcp("iscsi_tcp_state_change: TCP_CLOSE|TCP_CLOSE_WAIT\n");
1087                 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
1088         }
1089
1090         tcp_conn = conn->dd_data;
1091         old_state_change = tcp_conn->old_state_change;
1092
1093         read_unlock(&sk->sk_callback_lock);
1094
1095         old_state_change(sk);
1096 }
1097
1098 /**
1099  * iscsi_write_space - Called when more output buffer space is available
1100  * @sk: socket space is available for
1101  **/
1102 static void
1103 iscsi_write_space(struct sock *sk)
1104 {
1105         struct iscsi_conn *conn = (struct iscsi_conn*)sk->sk_user_data;
1106         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1107
1108         tcp_conn->old_write_space(sk);
1109         debug_tcp("iscsi_write_space: cid %d\n", conn->id);
1110         scsi_queue_work(conn->session->host, &conn->xmitwork);
1111 }
1112
1113 static void
1114 iscsi_conn_set_callbacks(struct iscsi_conn *conn)
1115 {
1116         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1117         struct sock *sk = tcp_conn->sock->sk;
1118
1119         /* assign new callbacks */
1120         write_lock_bh(&sk->sk_callback_lock);
1121         sk->sk_user_data = conn;
1122         tcp_conn->old_data_ready = sk->sk_data_ready;
1123         tcp_conn->old_state_change = sk->sk_state_change;
1124         tcp_conn->old_write_space = sk->sk_write_space;
1125         sk->sk_data_ready = iscsi_tcp_data_ready;
1126         sk->sk_state_change = iscsi_tcp_state_change;
1127         sk->sk_write_space = iscsi_write_space;
1128         write_unlock_bh(&sk->sk_callback_lock);
1129 }
1130
1131 static void
1132 iscsi_conn_restore_callbacks(struct iscsi_tcp_conn *tcp_conn)
1133 {
1134         struct sock *sk = tcp_conn->sock->sk;
1135
1136         /* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
1137         write_lock_bh(&sk->sk_callback_lock);
1138         sk->sk_user_data    = NULL;
1139         sk->sk_data_ready   = tcp_conn->old_data_ready;
1140         sk->sk_state_change = tcp_conn->old_state_change;
1141         sk->sk_write_space  = tcp_conn->old_write_space;
1142         sk->sk_no_check  = 0;
1143         write_unlock_bh(&sk->sk_callback_lock);
1144 }
1145
1146 /**
1147  * iscsi_xmit - TCP transmit
1148  **/
1149 static int
1150 iscsi_xmit(struct iscsi_conn *conn)
1151 {
1152         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1153         struct iscsi_segment *segment = &tcp_conn->out.segment;
1154         unsigned int consumed = 0;
1155         int rc = 0;
1156
1157         while (1) {
1158                 rc = iscsi_tcp_xmit_segment(tcp_conn, segment);
1159                 if (rc < 0) {
1160                         rc = ISCSI_ERR_XMIT_FAILED;
1161                         goto error;
1162                 }
1163                 if (rc == 0)
1164                         break;
1165
1166                 consumed += rc;
1167
1168                 if (segment->total_copied >= segment->total_size) {
1169                         if (segment->done != NULL) {
1170                                 rc = segment->done(tcp_conn, segment);
1171                                 if (rc != 0)
1172                                         goto error;
1173                         }
1174                 }
1175         }
1176
1177         debug_tcp("xmit %d bytes\n", consumed);
1178
1179         conn->txdata_octets += consumed;
1180         return consumed;
1181
1182 error:
1183         /* Transmit error. We could initiate error recovery
1184          * here. */
1185         debug_tcp("Error sending PDU, errno=%d\n", rc);
1186         iscsi_conn_failure(conn, rc);
1187         return -EIO;
1188 }
1189
1190 /**
1191  * iscsi_tcp_xmit_qlen - return the number of bytes queued for xmit
1192  */
1193 static inline int
1194 iscsi_tcp_xmit_qlen(struct iscsi_conn *conn)
1195 {
1196         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1197         struct iscsi_segment *segment = &tcp_conn->out.segment;
1198
1199         return segment->total_copied - segment->total_size;
1200 }
1201
1202 static inline int
1203 iscsi_tcp_flush(struct iscsi_conn *conn)
1204 {
1205         int rc;
1206
1207         while (iscsi_tcp_xmit_qlen(conn)) {
1208                 rc = iscsi_xmit(conn);
1209                 if (rc == 0)
1210                         return -EAGAIN;
1211                 if (rc < 0)
1212                         return rc;
1213         }
1214
1215         return 0;
1216 }
1217
1218 /*
1219  * This is called when we're done sending the header.
1220  * Simply copy the data_segment to the send segment, and return.
1221  */
1222 static int
1223 iscsi_tcp_send_hdr_done(struct iscsi_tcp_conn *tcp_conn,
1224                         struct iscsi_segment *segment)
1225 {
1226         tcp_conn->out.segment = tcp_conn->out.data_segment;
1227         debug_tcp("Header done. Next segment size %u total_size %u\n",
1228                   tcp_conn->out.segment.size, tcp_conn->out.segment.total_size);
1229         return 0;
1230 }
1231
1232 static void
1233 iscsi_tcp_send_hdr_prep(struct iscsi_conn *conn, void *hdr, size_t hdrlen)
1234 {
1235         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1236
1237         debug_tcp("%s(%p%s)\n", __func__, tcp_conn,
1238                         conn->hdrdgst_en? ", digest enabled" : "");
1239
1240         /* Clear the data segment - needs to be filled in by the
1241          * caller using iscsi_tcp_send_data_prep() */
1242         memset(&tcp_conn->out.data_segment, 0, sizeof(struct iscsi_segment));
1243
1244         /* If header digest is enabled, compute the CRC and
1245          * place the digest into the same buffer. We make
1246          * sure that both iscsi_tcp_task and mtask have
1247          * sufficient room.
1248          */
1249         if (conn->hdrdgst_en) {
1250                 iscsi_tcp_dgst_header(&tcp_conn->tx_hash, hdr, hdrlen,
1251                                       hdr + hdrlen);
1252                 hdrlen += ISCSI_DIGEST_SIZE;
1253         }
1254
1255         /* Remember header pointer for later, when we need
1256          * to decide whether there's a payload to go along
1257          * with the header. */
1258         tcp_conn->out.hdr = hdr;
1259
1260         iscsi_segment_init_linear(&tcp_conn->out.segment, hdr, hdrlen,
1261                                 iscsi_tcp_send_hdr_done, NULL);
1262 }
1263
1264 /*
1265  * Prepare the send buffer for the payload data.
1266  * Padding and checksumming will all be taken care
1267  * of by the iscsi_segment routines.
1268  */
1269 static int
1270 iscsi_tcp_send_data_prep(struct iscsi_conn *conn, struct scatterlist *sg,
1271                          unsigned int count, unsigned int offset,
1272                          unsigned int len)
1273 {
1274         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1275         struct hash_desc *tx_hash = NULL;
1276         unsigned int hdr_spec_len;
1277
1278         debug_tcp("%s(%p, offset=%d, datalen=%d%s)\n", __func__,
1279                         tcp_conn, offset, len,
1280                         conn->datadgst_en? ", digest enabled" : "");
1281
1282         /* Make sure the datalen matches what the caller
1283            said he would send. */
1284         hdr_spec_len = ntoh24(tcp_conn->out.hdr->dlength);
1285         WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len));
1286
1287         if (conn->datadgst_en)
1288                 tx_hash = &tcp_conn->tx_hash;
1289
1290         return iscsi_segment_seek_sg(&tcp_conn->out.data_segment,
1291                                    sg, count, offset, len,
1292                                    NULL, tx_hash);
1293 }
1294
1295 static void
1296 iscsi_tcp_send_linear_data_prepare(struct iscsi_conn *conn, void *data,
1297                                    size_t len)
1298 {
1299         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1300         struct hash_desc *tx_hash = NULL;
1301         unsigned int hdr_spec_len;
1302
1303         debug_tcp("%s(%p, datalen=%d%s)\n", __func__, tcp_conn, len,
1304                   conn->datadgst_en? ", digest enabled" : "");
1305
1306         /* Make sure the datalen matches what the caller
1307            said he would send. */
1308         hdr_spec_len = ntoh24(tcp_conn->out.hdr->dlength);
1309         WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len));
1310
1311         if (conn->datadgst_en)
1312                 tx_hash = &tcp_conn->tx_hash;
1313
1314         iscsi_segment_init_linear(&tcp_conn->out.data_segment,
1315                                 data, len, NULL, tx_hash);
1316 }
1317
1318 /**
1319  * iscsi_solicit_data_cont - initialize next Data-Out
1320  * @conn: iscsi connection
1321  * @task: scsi command task
1322  * @r2t: R2T info
1323  * @left: bytes left to transfer
1324  *
1325  * Notes:
1326  *      Initialize next Data-Out within this R2T sequence and continue
1327  *      to process next Scatter-Gather element(if any) of this SCSI command.
1328  *
1329  *      Called under connection lock.
1330  **/
1331 static int
1332 iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_task *task,
1333                         struct iscsi_r2t_info *r2t)
1334 {
1335         struct iscsi_data *hdr;
1336         int new_offset, left;
1337
1338         BUG_ON(r2t->data_length - r2t->sent < 0);
1339         left = r2t->data_length - r2t->sent;
1340         if (left == 0)
1341                 return 0;
1342
1343         hdr = &r2t->dtask.hdr;
1344         memset(hdr, 0, sizeof(struct iscsi_data));
1345         hdr->ttt = r2t->ttt;
1346         hdr->datasn = cpu_to_be32(r2t->solicit_datasn);
1347         r2t->solicit_datasn++;
1348         hdr->opcode = ISCSI_OP_SCSI_DATA_OUT;
1349         memcpy(hdr->lun, task->hdr->lun, sizeof(hdr->lun));
1350         hdr->itt = task->hdr->itt;
1351         hdr->exp_statsn = r2t->exp_statsn;
1352         new_offset = r2t->data_offset + r2t->sent;
1353         hdr->offset = cpu_to_be32(new_offset);
1354         if (left > conn->max_xmit_dlength) {
1355                 hton24(hdr->dlength, conn->max_xmit_dlength);
1356                 r2t->data_count = conn->max_xmit_dlength;
1357         } else {
1358                 hton24(hdr->dlength, left);
1359                 r2t->data_count = left;
1360                 hdr->flags = ISCSI_FLAG_CMD_FINAL;
1361         }
1362
1363         conn->dataout_pdus_cnt++;
1364         return 1;
1365 }
1366
1367 /**
1368  * iscsi_tcp_task - Initialize iSCSI SCSI_READ or SCSI_WRITE commands
1369  * @conn: iscsi connection
1370  * @task: scsi command task
1371  * @sc: scsi command
1372  **/
1373 static int
1374 iscsi_tcp_task_init(struct iscsi_task *task)
1375 {
1376         struct iscsi_tcp_task *tcp_task = task->dd_data;
1377         struct iscsi_conn *conn = task->conn;
1378         struct scsi_cmnd *sc = task->sc;
1379         int err;
1380
1381         if (!sc) {
1382                 /*
1383                  * mgmt tasks do not have a scatterlist since they come
1384                  * in from the iscsi interface.
1385                  */
1386                 debug_scsi("mtask deq [cid %d itt 0x%x]\n", conn->id,
1387                            task->itt);
1388
1389                 /* Prepare PDU, optionally w/ immediate data */
1390                 iscsi_tcp_send_hdr_prep(conn, task->hdr, sizeof(*task->hdr));
1391
1392                 /* If we have immediate data, attach a payload */
1393                 if (task->data_count)
1394                         iscsi_tcp_send_linear_data_prepare(conn, task->data,
1395                                                            task->data_count);
1396                 return 0;
1397         }
1398
1399         BUG_ON(__kfifo_len(tcp_task->r2tqueue));
1400         tcp_task->sent = 0;
1401         tcp_task->exp_datasn = 0;
1402
1403         /* Prepare PDU, optionally w/ immediate data */
1404         debug_scsi("task deq [cid %d itt 0x%x imm %d unsol %d]\n",
1405                     conn->id, task->itt, task->imm_count,
1406                     task->unsol_count);
1407         iscsi_tcp_send_hdr_prep(conn, task->hdr, task->hdr_len);
1408
1409         if (!task->imm_count)
1410                 return 0;
1411
1412         /* If we have immediate data, attach a payload */
1413         err = iscsi_tcp_send_data_prep(conn, scsi_out(sc)->table.sgl,
1414                                        scsi_out(sc)->table.nents,
1415                                        0, task->imm_count);
1416         if (err)
1417                 return err;
1418         tcp_task->sent += task->imm_count;
1419         task->imm_count = 0;
1420         return 0;
1421 }
1422
1423 /*
1424  * iscsi_tcp_task_xmit - xmit normal PDU task
1425  * @task: iscsi command task
1426  *
1427  * We're expected to return 0 when everything was transmitted succesfully,
1428  * -EAGAIN if there's still data in the queue, or != 0 for any other kind
1429  * of error.
1430  */
1431 static int
1432 iscsi_tcp_task_xmit(struct iscsi_task *task)
1433 {
1434         struct iscsi_conn *conn = task->conn;
1435         struct iscsi_tcp_task *tcp_task = task->dd_data;
1436         struct scsi_cmnd *sc = task->sc;
1437         struct scsi_data_buffer *sdb;
1438         int rc = 0;
1439
1440 flush:
1441         /* Flush any pending data first. */
1442         rc = iscsi_tcp_flush(conn);
1443         if (rc < 0)
1444                 return rc;
1445
1446         /* mgmt command */
1447         if (!sc) {
1448                 if (task->hdr->itt == RESERVED_ITT)
1449                         iscsi_put_task(task);
1450                 return 0;
1451         }
1452
1453         /* Are we done already? */
1454         if (sc->sc_data_direction != DMA_TO_DEVICE)
1455                 return 0;
1456
1457         sdb = scsi_out(sc);
1458         if (task->unsol_count != 0) {
1459                 struct iscsi_data *hdr = &tcp_task->unsol_dtask.hdr;
1460
1461                 /* Prepare a header for the unsolicited PDU.
1462                  * The amount of data we want to send will be
1463                  * in task->data_count.
1464                  * FIXME: return the data count instead.
1465                  */
1466                 iscsi_prep_unsolicit_data_pdu(task, hdr);
1467
1468                 debug_tcp("unsol dout [itt 0x%x doff %d dlen %d]\n",
1469                                 task->itt, tcp_task->sent, task->data_count);
1470
1471                 iscsi_tcp_send_hdr_prep(conn, hdr, sizeof(*hdr));
1472                 rc = iscsi_tcp_send_data_prep(conn, sdb->table.sgl,
1473                                               sdb->table.nents, tcp_task->sent,
1474                                               task->data_count);
1475                 if (rc)
1476                         goto fail;
1477                 tcp_task->sent += task->data_count;
1478                 task->unsol_count -= task->data_count;
1479                 goto flush;
1480         } else {
1481                 struct iscsi_session *session = conn->session;
1482                 struct iscsi_r2t_info *r2t;
1483
1484                 /* All unsolicited PDUs sent. Check for solicited PDUs.
1485                  */
1486                 spin_lock_bh(&session->lock);
1487                 r2t = tcp_task->r2t;
1488                 if (r2t != NULL) {
1489                         /* Continue with this R2T? */
1490                         if (!iscsi_solicit_data_cont(conn, task, r2t)) {
1491                                 debug_scsi("  done with r2t %p\n", r2t);
1492
1493                                 __kfifo_put(tcp_task->r2tpool.queue,
1494                                             (void*)&r2t, sizeof(void*));
1495                                 tcp_task->r2t = r2t = NULL;
1496                         }
1497                 }
1498
1499                 if (r2t == NULL) {
1500                         __kfifo_get(tcp_task->r2tqueue, (void*)&tcp_task->r2t,
1501                                     sizeof(void*));
1502                         r2t = tcp_task->r2t;
1503                 }
1504                 spin_unlock_bh(&session->lock);
1505
1506                 /* Waiting for more R2Ts to arrive. */
1507                 if (r2t == NULL) {
1508                         debug_tcp("no R2Ts yet\n");
1509                         return 0;
1510                 }
1511
1512                 debug_scsi("sol dout %p [dsn %d itt 0x%x doff %d dlen %d]\n",
1513                         r2t, r2t->solicit_datasn - 1, task->itt,
1514                         r2t->data_offset + r2t->sent, r2t->data_count);
1515
1516                 iscsi_tcp_send_hdr_prep(conn, &r2t->dtask.hdr,
1517                                         sizeof(struct iscsi_hdr));
1518
1519                 rc = iscsi_tcp_send_data_prep(conn, sdb->table.sgl,
1520                                               sdb->table.nents,
1521                                               r2t->data_offset + r2t->sent,
1522                                               r2t->data_count);
1523                 if (rc)
1524                         goto fail;
1525                 tcp_task->sent += r2t->data_count;
1526                 r2t->sent += r2t->data_count;
1527                 goto flush;
1528         }
1529         return 0;
1530 fail:
1531         iscsi_conn_failure(conn, rc);
1532         return -EIO;
1533 }
1534
1535 static struct iscsi_cls_conn *
1536 iscsi_tcp_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx)
1537 {
1538         struct iscsi_conn *conn;
1539         struct iscsi_cls_conn *cls_conn;
1540         struct iscsi_tcp_conn *tcp_conn;
1541
1542         cls_conn = iscsi_conn_setup(cls_session, sizeof(*tcp_conn), conn_idx);
1543         if (!cls_conn)
1544                 return NULL;
1545         conn = cls_conn->dd_data;
1546         /*
1547          * due to strange issues with iser these are not set
1548          * in iscsi_conn_setup
1549          */
1550         conn->max_recv_dlength = ISCSI_DEF_MAX_RECV_SEG_LEN;
1551
1552         tcp_conn = conn->dd_data;
1553         tcp_conn->iscsi_conn = conn;
1554
1555         tcp_conn->tx_hash.tfm = crypto_alloc_hash("crc32c", 0,
1556                                                   CRYPTO_ALG_ASYNC);
1557         tcp_conn->tx_hash.flags = 0;
1558         if (IS_ERR(tcp_conn->tx_hash.tfm))
1559                 goto free_conn;
1560
1561         tcp_conn->rx_hash.tfm = crypto_alloc_hash("crc32c", 0,
1562                                                   CRYPTO_ALG_ASYNC);
1563         tcp_conn->rx_hash.flags = 0;
1564         if (IS_ERR(tcp_conn->rx_hash.tfm))
1565                 goto free_tx_tfm;
1566
1567         return cls_conn;
1568
1569 free_tx_tfm:
1570         crypto_free_hash(tcp_conn->tx_hash.tfm);
1571 free_conn:
1572         iscsi_conn_printk(KERN_ERR, conn,
1573                           "Could not create connection due to crc32c "
1574                           "loading error. Make sure the crc32c "
1575                           "module is built as a module or into the "
1576                           "kernel\n");
1577         iscsi_conn_teardown(cls_conn);
1578         return NULL;
1579 }
1580
1581 static void
1582 iscsi_tcp_release_conn(struct iscsi_conn *conn)
1583 {
1584         struct iscsi_session *session = conn->session;
1585         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1586         struct socket *sock = tcp_conn->sock;
1587
1588         if (!sock)
1589                 return;
1590
1591         sock_hold(sock->sk);
1592         iscsi_conn_restore_callbacks(tcp_conn);
1593         sock_put(sock->sk);
1594
1595         spin_lock_bh(&session->lock);
1596         tcp_conn->sock = NULL;
1597         spin_unlock_bh(&session->lock);
1598         sockfd_put(sock);
1599 }
1600
1601 static void
1602 iscsi_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
1603 {
1604         struct iscsi_conn *conn = cls_conn->dd_data;
1605         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1606
1607         iscsi_tcp_release_conn(conn);
1608
1609         if (tcp_conn->tx_hash.tfm)
1610                 crypto_free_hash(tcp_conn->tx_hash.tfm);
1611         if (tcp_conn->rx_hash.tfm)
1612                 crypto_free_hash(tcp_conn->rx_hash.tfm);
1613
1614         iscsi_conn_teardown(cls_conn);
1615 }
1616
1617 static void
1618 iscsi_tcp_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
1619 {
1620         struct iscsi_conn *conn = cls_conn->dd_data;
1621         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1622
1623         /* userspace may have goofed up and not bound us */
1624         if (!tcp_conn->sock)
1625                 return;
1626         /*
1627          * Make sure our recv side is stopped.
1628          * Older tools called conn stop before ep_disconnect
1629          * so IO could still be coming in.
1630          */
1631         write_lock_bh(&tcp_conn->sock->sk->sk_callback_lock);
1632         set_bit(ISCSI_SUSPEND_BIT, &conn->suspend_rx);
1633         write_unlock_bh(&tcp_conn->sock->sk->sk_callback_lock);
1634
1635         iscsi_conn_stop(cls_conn, flag);
1636         iscsi_tcp_release_conn(conn);
1637 }
1638
1639 static int iscsi_tcp_get_addr(struct iscsi_conn *conn, struct socket *sock,
1640                               char *buf, int *port,
1641                               int (*getname)(struct socket *, struct sockaddr *,
1642                                         int *addrlen))
1643 {
1644         struct sockaddr_storage *addr;
1645         struct sockaddr_in6 *sin6;
1646         struct sockaddr_in *sin;
1647         int rc = 0, len;
1648
1649         addr = kmalloc(sizeof(*addr), GFP_KERNEL);
1650         if (!addr)
1651                 return -ENOMEM;
1652
1653         if (getname(sock, (struct sockaddr *) addr, &len)) {
1654                 rc = -ENODEV;
1655                 goto free_addr;
1656         }
1657
1658         switch (addr->ss_family) {
1659         case AF_INET:
1660                 sin = (struct sockaddr_in *)addr;
1661                 spin_lock_bh(&conn->session->lock);
1662                 sprintf(buf, "%pI4", &sin->sin_addr.s_addr);
1663                 *port = be16_to_cpu(sin->sin_port);
1664                 spin_unlock_bh(&conn->session->lock);
1665                 break;
1666         case AF_INET6:
1667                 sin6 = (struct sockaddr_in6 *)addr;
1668                 spin_lock_bh(&conn->session->lock);
1669                 sprintf(buf, "%pI6", &sin6->sin6_addr);
1670                 *port = be16_to_cpu(sin6->sin6_port);
1671                 spin_unlock_bh(&conn->session->lock);
1672                 break;
1673         }
1674 free_addr:
1675         kfree(addr);
1676         return rc;
1677 }
1678
1679 static int
1680 iscsi_tcp_conn_bind(struct iscsi_cls_session *cls_session,
1681                     struct iscsi_cls_conn *cls_conn, uint64_t transport_eph,
1682                     int is_leading)
1683 {
1684         struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
1685         struct iscsi_host *ihost = shost_priv(shost);
1686         struct iscsi_conn *conn = cls_conn->dd_data;
1687         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1688         struct sock *sk;
1689         struct socket *sock;
1690         int err;
1691
1692         /* lookup for existing socket */
1693         sock = sockfd_lookup((int)transport_eph, &err);
1694         if (!sock) {
1695                 iscsi_conn_printk(KERN_ERR, conn,
1696                                   "sockfd_lookup failed %d\n", err);
1697                 return -EEXIST;
1698         }
1699         /*
1700          * copy these values now because if we drop the session
1701          * userspace may still want to query the values since we will
1702          * be using them for the reconnect
1703          */
1704         err = iscsi_tcp_get_addr(conn, sock, conn->portal_address,
1705                                  &conn->portal_port, kernel_getpeername);
1706         if (err)
1707                 goto free_socket;
1708
1709         err = iscsi_tcp_get_addr(conn, sock, ihost->local_address,
1710                                 &ihost->local_port, kernel_getsockname);
1711         if (err)
1712                 goto free_socket;
1713
1714         err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
1715         if (err)
1716                 goto free_socket;
1717
1718         /* bind iSCSI connection and socket */
1719         tcp_conn->sock = sock;
1720
1721         /* setup Socket parameters */
1722         sk = sock->sk;
1723         sk->sk_reuse = 1;
1724         sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
1725         sk->sk_allocation = GFP_ATOMIC;
1726
1727         iscsi_conn_set_callbacks(conn);
1728         tcp_conn->sendpage = tcp_conn->sock->ops->sendpage;
1729         /*
1730          * set receive state machine into initial state
1731          */
1732         iscsi_tcp_hdr_recv_prep(tcp_conn);
1733         return 0;
1734
1735 free_socket:
1736         sockfd_put(sock);
1737         return err;
1738 }
1739
1740 static int
1741 iscsi_r2tpool_alloc(struct iscsi_session *session)
1742 {
1743         int i;
1744         int cmd_i;
1745
1746         /*
1747          * initialize per-task: R2T pool and xmit queue
1748          */
1749         for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
1750                 struct iscsi_task *task = session->cmds[cmd_i];
1751                 struct iscsi_tcp_task *tcp_task = task->dd_data;
1752
1753                 /*
1754                  * pre-allocated x4 as much r2ts to handle race when
1755                  * target acks DataOut faster than we data_xmit() queues
1756                  * could replenish r2tqueue.
1757                  */
1758
1759                 /* R2T pool */
1760                 if (iscsi_pool_init(&tcp_task->r2tpool, session->max_r2t * 4, NULL,
1761                                     sizeof(struct iscsi_r2t_info))) {
1762                         goto r2t_alloc_fail;
1763                 }
1764
1765                 /* R2T xmit queue */
1766                 tcp_task->r2tqueue = kfifo_alloc(
1767                       session->max_r2t * 4 * sizeof(void*), GFP_KERNEL, NULL);
1768                 if (tcp_task->r2tqueue == ERR_PTR(-ENOMEM)) {
1769                         iscsi_pool_free(&tcp_task->r2tpool);
1770                         goto r2t_alloc_fail;
1771                 }
1772         }
1773
1774         return 0;
1775
1776 r2t_alloc_fail:
1777         for (i = 0; i < cmd_i; i++) {
1778                 struct iscsi_task *task = session->cmds[i];
1779                 struct iscsi_tcp_task *tcp_task = task->dd_data;
1780
1781                 kfifo_free(tcp_task->r2tqueue);
1782                 iscsi_pool_free(&tcp_task->r2tpool);
1783         }
1784         return -ENOMEM;
1785 }
1786
1787 static void
1788 iscsi_r2tpool_free(struct iscsi_session *session)
1789 {
1790         int i;
1791
1792         for (i = 0; i < session->cmds_max; i++) {
1793                 struct iscsi_task *task = session->cmds[i];
1794                 struct iscsi_tcp_task *tcp_task = task->dd_data;
1795
1796                 kfifo_free(tcp_task->r2tqueue);
1797                 iscsi_pool_free(&tcp_task->r2tpool);
1798         }
1799 }
1800
1801 static int
1802 iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
1803                      char *buf, int buflen)
1804 {
1805         struct iscsi_conn *conn = cls_conn->dd_data;
1806         struct iscsi_session *session = conn->session;
1807         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1808         int value;
1809
1810         switch(param) {
1811         case ISCSI_PARAM_HDRDGST_EN:
1812                 iscsi_set_param(cls_conn, param, buf, buflen);
1813                 break;
1814         case ISCSI_PARAM_DATADGST_EN:
1815                 iscsi_set_param(cls_conn, param, buf, buflen);
1816                 tcp_conn->sendpage = conn->datadgst_en ?
1817                         sock_no_sendpage : tcp_conn->sock->ops->sendpage;
1818                 break;
1819         case ISCSI_PARAM_MAX_R2T:
1820                 sscanf(buf, "%d", &value);
1821                 if (value <= 0 || !is_power_of_2(value))
1822                         return -EINVAL;
1823                 if (session->max_r2t == value)
1824                         break;
1825                 iscsi_r2tpool_free(session);
1826                 iscsi_set_param(cls_conn, param, buf, buflen);
1827                 if (iscsi_r2tpool_alloc(session))
1828                         return -ENOMEM;
1829                 break;
1830         default:
1831                 return iscsi_set_param(cls_conn, param, buf, buflen);
1832         }
1833
1834         return 0;
1835 }
1836
1837 static int
1838 iscsi_tcp_conn_get_param(struct iscsi_cls_conn *cls_conn,
1839                          enum iscsi_param param, char *buf)
1840 {
1841         struct iscsi_conn *conn = cls_conn->dd_data;
1842         int len;
1843
1844         switch(param) {
1845         case ISCSI_PARAM_CONN_PORT:
1846                 spin_lock_bh(&conn->session->lock);
1847                 len = sprintf(buf, "%hu\n", conn->portal_port);
1848                 spin_unlock_bh(&conn->session->lock);
1849                 break;
1850         case ISCSI_PARAM_CONN_ADDRESS:
1851                 spin_lock_bh(&conn->session->lock);
1852                 len = sprintf(buf, "%s\n", conn->portal_address);
1853                 spin_unlock_bh(&conn->session->lock);
1854                 break;
1855         default:
1856                 return iscsi_conn_get_param(cls_conn, param, buf);
1857         }
1858
1859         return len;
1860 }
1861
1862 static void
1863 iscsi_conn_get_stats(struct iscsi_cls_conn *cls_conn, struct iscsi_stats *stats)
1864 {
1865         struct iscsi_conn *conn = cls_conn->dd_data;
1866         struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
1867
1868         stats->txdata_octets = conn->txdata_octets;
1869         stats->rxdata_octets = conn->rxdata_octets;
1870         stats->scsicmd_pdus = conn->scsicmd_pdus_cnt;
1871         stats->dataout_pdus = conn->dataout_pdus_cnt;
1872         stats->scsirsp_pdus = conn->scsirsp_pdus_cnt;
1873         stats->datain_pdus = conn->datain_pdus_cnt;
1874         stats->r2t_pdus = conn->r2t_pdus_cnt;
1875         stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt;
1876         stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt;
1877         stats->custom_length = 3;
1878         strcpy(stats->custom[0].desc, "tx_sendpage_failures");
1879         stats->custom[0].value = tcp_conn->sendpage_failures_cnt;
1880         strcpy(stats->custom[1].desc, "rx_discontiguous_hdr");
1881         stats->custom[1].value = tcp_conn->discontiguous_hdr_cnt;
1882         strcpy(stats->custom[2].desc, "eh_abort_cnt");
1883         stats->custom[2].value = conn->eh_abort_cnt;
1884 }
1885
1886 static struct iscsi_cls_session *
1887 iscsi_tcp_session_create(struct iscsi_endpoint *ep, uint16_t cmds_max,
1888                          uint16_t qdepth, uint32_t initial_cmdsn,
1889                          uint32_t *hostno)
1890 {
1891         struct iscsi_cls_session *cls_session;
1892         struct iscsi_session *session;
1893         struct Scsi_Host *shost;
1894         int cmd_i;
1895
1896         if (ep) {
1897                 printk(KERN_ERR "iscsi_tcp: invalid ep %p.\n", ep);
1898                 return NULL;
1899         }
1900
1901         shost = iscsi_host_alloc(&iscsi_sht, 0, qdepth);
1902         if (!shost)
1903                 return NULL;
1904         shost->transportt = iscsi_tcp_scsi_transport;
1905         shost->max_lun = iscsi_max_lun;
1906         shost->max_id = 0;
1907         shost->max_channel = 0;
1908         shost->max_cmd_len = SCSI_MAX_VARLEN_CDB_SIZE;
1909
1910         if (iscsi_host_add(shost, NULL))
1911                 goto free_host;
1912         *hostno = shost->host_no;
1913
1914         cls_session = iscsi_session_setup(&iscsi_tcp_transport, shost, cmds_max,
1915                                           sizeof(struct iscsi_tcp_task),
1916                                           initial_cmdsn, 0);
1917         if (!cls_session)
1918                 goto remove_host;
1919         session = cls_session->dd_data;
1920
1921         shost->can_queue = session->scsi_cmds_max;
1922         for (cmd_i = 0; cmd_i < session->cmds_max; cmd_i++) {
1923                 struct iscsi_task *task = session->cmds[cmd_i];
1924                 struct iscsi_tcp_task *tcp_task = task->dd_data;
1925
1926                 task->hdr = &tcp_task->hdr.cmd_hdr;
1927                 task->hdr_max = sizeof(tcp_task->hdr) - ISCSI_DIGEST_SIZE;
1928         }
1929
1930         if (iscsi_r2tpool_alloc(session))
1931                 goto remove_session;
1932         return cls_session;
1933
1934 remove_session:
1935         iscsi_session_teardown(cls_session);
1936 remove_host:
1937         iscsi_host_remove(shost);
1938 free_host:
1939         iscsi_host_free(shost);
1940         return NULL;
1941 }
1942
1943 static void iscsi_tcp_session_destroy(struct iscsi_cls_session *cls_session)
1944 {
1945         struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
1946
1947         iscsi_r2tpool_free(cls_session->dd_data);
1948         iscsi_session_teardown(cls_session);
1949
1950         iscsi_host_remove(shost);
1951         iscsi_host_free(shost);
1952 }
1953
1954 static int iscsi_tcp_slave_configure(struct scsi_device *sdev)
1955 {
1956         blk_queue_bounce_limit(sdev->request_queue, BLK_BOUNCE_ANY);
1957         blk_queue_dma_alignment(sdev->request_queue, 0);
1958         return 0;
1959 }
1960
1961 static struct scsi_host_template iscsi_sht = {
1962         .module                 = THIS_MODULE,
1963         .name                   = "iSCSI Initiator over TCP/IP",
1964         .queuecommand           = iscsi_queuecommand,
1965         .change_queue_depth     = iscsi_change_queue_depth,
1966         .can_queue              = ISCSI_DEF_XMIT_CMDS_MAX - 1,
1967         .sg_tablesize           = 4096,
1968         .max_sectors            = 0xFFFF,
1969         .cmd_per_lun            = ISCSI_DEF_CMD_PER_LUN,
1970         .eh_abort_handler       = iscsi_eh_abort,
1971         .eh_device_reset_handler= iscsi_eh_device_reset,
1972         .eh_target_reset_handler= iscsi_eh_target_reset,
1973         .use_clustering         = DISABLE_CLUSTERING,
1974         .slave_configure        = iscsi_tcp_slave_configure,
1975         .proc_name              = "iscsi_tcp",
1976         .this_id                = -1,
1977 };
1978
1979 static struct iscsi_transport iscsi_tcp_transport = {
1980         .owner                  = THIS_MODULE,
1981         .name                   = "tcp",
1982         .caps                   = CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
1983                                   | CAP_DATADGST,
1984         .param_mask             = ISCSI_MAX_RECV_DLENGTH |
1985                                   ISCSI_MAX_XMIT_DLENGTH |
1986                                   ISCSI_HDRDGST_EN |
1987                                   ISCSI_DATADGST_EN |
1988                                   ISCSI_INITIAL_R2T_EN |
1989                                   ISCSI_MAX_R2T |
1990                                   ISCSI_IMM_DATA_EN |
1991                                   ISCSI_FIRST_BURST |
1992                                   ISCSI_MAX_BURST |
1993                                   ISCSI_PDU_INORDER_EN |
1994                                   ISCSI_DATASEQ_INORDER_EN |
1995                                   ISCSI_ERL |
1996                                   ISCSI_CONN_PORT |
1997                                   ISCSI_CONN_ADDRESS |
1998                                   ISCSI_EXP_STATSN |
1999                                   ISCSI_PERSISTENT_PORT |
2000                                   ISCSI_PERSISTENT_ADDRESS |
2001                                   ISCSI_TARGET_NAME | ISCSI_TPGT |
2002                                   ISCSI_USERNAME | ISCSI_PASSWORD |
2003                                   ISCSI_USERNAME_IN | ISCSI_PASSWORD_IN |
2004                                   ISCSI_FAST_ABORT | ISCSI_ABORT_TMO |
2005                                   ISCSI_LU_RESET_TMO |
2006                                   ISCSI_PING_TMO | ISCSI_RECV_TMO |
2007                                   ISCSI_IFACE_NAME | ISCSI_INITIATOR_NAME,
2008         .host_param_mask        = ISCSI_HOST_HWADDRESS | ISCSI_HOST_IPADDRESS |
2009                                   ISCSI_HOST_INITIATOR_NAME |
2010                                   ISCSI_HOST_NETDEV_NAME,
2011         /* session management */
2012         .create_session         = iscsi_tcp_session_create,
2013         .destroy_session        = iscsi_tcp_session_destroy,
2014         /* connection management */
2015         .create_conn            = iscsi_tcp_conn_create,
2016         .bind_conn              = iscsi_tcp_conn_bind,
2017         .destroy_conn           = iscsi_tcp_conn_destroy,
2018         .set_param              = iscsi_conn_set_param,
2019         .get_conn_param         = iscsi_tcp_conn_get_param,
2020         .get_session_param      = iscsi_session_get_param,
2021         .start_conn             = iscsi_conn_start,
2022         .stop_conn              = iscsi_tcp_conn_stop,
2023         /* iscsi host params */
2024         .get_host_param         = iscsi_host_get_param,
2025         .set_host_param         = iscsi_host_set_param,
2026         /* IO */
2027         .send_pdu               = iscsi_conn_send_pdu,
2028         .get_stats              = iscsi_conn_get_stats,
2029         .init_task              = iscsi_tcp_task_init,
2030         .xmit_task              = iscsi_tcp_task_xmit,
2031         .cleanup_task           = iscsi_tcp_cleanup_task,
2032         /* recovery */
2033         .session_recovery_timedout = iscsi_session_recovery_timedout,
2034 };
2035
2036 static int __init
2037 iscsi_tcp_init(void)
2038 {
2039         if (iscsi_max_lun < 1) {
2040                 printk(KERN_ERR "iscsi_tcp: Invalid max_lun value of %u\n",
2041                        iscsi_max_lun);
2042                 return -EINVAL;
2043         }
2044
2045         iscsi_tcp_scsi_transport = iscsi_register_transport(
2046                                                         &iscsi_tcp_transport);
2047         if (!iscsi_tcp_scsi_transport)
2048                 return -ENODEV;
2049
2050         return 0;
2051 }
2052
2053 static void __exit
2054 iscsi_tcp_exit(void)
2055 {
2056         iscsi_unregister_transport(&iscsi_tcp_transport);
2057 }
2058
2059 module_init(iscsi_tcp_init);
2060 module_exit(iscsi_tcp_exit);