- debug_scsi("mtask deq [cid %d state %x itt 0x%x]\n",
- conn->id, mtask->xmstate, mtask->itt);
-
- if (mtask->xmstate & XMSTATE_IMM_HDR) {
- mtask->xmstate &= ~XMSTATE_IMM_HDR;
- if (mtask->data_count)
- mtask->xmstate |= XMSTATE_IMM_DATA;
- if (conn->c_stage != ISCSI_CONN_INITIAL_STAGE &&
- conn->stop_stage != STOP_CONN_RECOVER &&
- conn->hdrdgst_en)
- iscsi_hdr_digest(conn, &mtask->headbuf,
- (u8*)mtask->hdrext);
- if (iscsi_sendhdr(conn, &mtask->headbuf, mtask->data_count)) {
- mtask->xmstate |= XMSTATE_IMM_HDR;
- if (mtask->data_count)
- mtask->xmstate &= ~XMSTATE_IMM_DATA;
- return -EAGAIN;
- }
- }
-
- if (mtask->xmstate & XMSTATE_IMM_DATA) {
- BUG_ON(!mtask->data_count);
- mtask->xmstate &= ~XMSTATE_IMM_DATA;
- /* FIXME: implement.
- * Virtual buffer could be spreaded across multiple pages...
- */
- do {
- if (iscsi_sendpage(conn, &mtask->sendbuf,
- &mtask->data_count, &mtask->sent)) {
- mtask->xmstate |= XMSTATE_IMM_DATA;
- return -EAGAIN;
- }
- } while (mtask->data_count);
- }
-
- BUG_ON(mtask->xmstate != XMSTATE_IDLE);
- return 0;
-}
-
-static inline int
-handle_xmstate_r_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
-{
- ctask->xmstate &= ~XMSTATE_R_HDR;
- if (conn->hdrdgst_en)
- iscsi_hdr_digest(conn, &ctask->headbuf, (u8*)ctask->hdrext);
- if (!iscsi_sendhdr(conn, &ctask->headbuf, 0)) {
- BUG_ON(ctask->xmstate != XMSTATE_IDLE);
- return 0; /* wait for Data-In */
- }
- ctask->xmstate |= XMSTATE_R_HDR;
- return -EAGAIN;
-}
-
-static inline int
-handle_xmstate_w_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
-{
- ctask->xmstate &= ~XMSTATE_W_HDR;
- if (conn->hdrdgst_en)
- iscsi_hdr_digest(conn, &ctask->headbuf, (u8*)ctask->hdrext);
- if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->imm_count)) {
- ctask->xmstate |= XMSTATE_W_HDR;
- return -EAGAIN;
- }
- return 0;
-}
-
-static inline int
-handle_xmstate_data_digest(struct iscsi_conn *conn,
- struct iscsi_cmd_task *ctask)
-{
- ctask->xmstate &= ~XMSTATE_DATA_DIGEST;
- debug_tcp("resent data digest 0x%x\n", ctask->datadigest);
- if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
- &ctask->datadigest, 0)) {
- ctask->xmstate |= XMSTATE_DATA_DIGEST;
- debug_tcp("resent data digest 0x%x fail!\n",
- ctask->datadigest);
- return -EAGAIN;
- }
- return 0;
-}
-
-static inline int
-handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
-{
- BUG_ON(!ctask->imm_count);
- ctask->xmstate &= ~XMSTATE_IMM_DATA;
-
- if (conn->datadgst_en) {
- iscsi_data_digest_init(conn, ctask);
- ctask->immdigest = 0;
- }
-
- for (;;) {
- if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->imm_count,
- &ctask->sent)) {
- ctask->xmstate |= XMSTATE_IMM_DATA;
- if (conn->datadgst_en) {
- crypto_digest_final(conn->data_tx_tfm,
- (u8*)&ctask->immdigest);
- debug_tcp("tx imm sendpage fail 0x%x\n",
- ctask->datadigest);
- }
- return -EAGAIN;
- }
- if (conn->datadgst_en)
- iscsi_buf_data_digest_update(conn, &ctask->sendbuf);
-
- if (!ctask->imm_count)
- break;
- iscsi_buf_init_sg(&ctask->sendbuf,
- &ctask->sg[ctask->sg_count++]);
- }
-
- if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) {
- if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
- &ctask->immdigest, 1)) {
- debug_tcp("sending imm digest 0x%x fail!\n",
- ctask->immdigest);
- return -EAGAIN;
- }
- debug_tcp("sending imm digest 0x%x\n", ctask->immdigest);
- }
-
- return 0;
-}
-
-static inline int
-handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
-{
- struct iscsi_data_task *dtask;
-
- ctask->xmstate |= XMSTATE_UNS_DATA;
- if (ctask->xmstate & XMSTATE_UNS_INIT) {
- iscsi_unsolicit_data_init(conn, ctask);
- BUG_ON(!ctask->dtask);
- dtask = ctask->dtask;
- if (conn->hdrdgst_en)
- iscsi_hdr_digest(conn, &ctask->headbuf,
- (u8*)dtask->hdrext);
- ctask->xmstate &= ~XMSTATE_UNS_INIT;
- }
- if (iscsi_sendhdr(conn, &ctask->headbuf, ctask->data_count)) {
- ctask->xmstate &= ~XMSTATE_UNS_DATA;
- ctask->xmstate |= XMSTATE_UNS_HDR;
- return -EAGAIN;
- }
-
- debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n",
- ctask->itt, ctask->unsol_count, ctask->sent);
- return 0;
-}
-
-static inline int
-handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
-{
- struct iscsi_data_task *dtask = ctask->dtask;
-
- BUG_ON(!ctask->data_count);
- ctask->xmstate &= ~XMSTATE_UNS_DATA;
-
- if (conn->datadgst_en) {
- iscsi_data_digest_init(conn, ctask);
- dtask->digest = 0;
- }
-
- for (;;) {
- int start = ctask->sent;
-
- if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->data_count,
- &ctask->sent)) {
- ctask->unsol_count -= ctask->sent - start;
- ctask->xmstate |= XMSTATE_UNS_DATA;
- /* will continue with this ctask later.. */
- if (conn->datadgst_en) {
- crypto_digest_final(conn->data_tx_tfm,
- (u8 *)&dtask->digest);
- debug_tcp("tx uns data fail 0x%x\n",
- dtask->digest);
- }
- return -EAGAIN;
- }
-
- BUG_ON(ctask->sent > ctask->total_length);
- ctask->unsol_count -= ctask->sent - start;
-
- /*
- * XXX:we may run here with un-initial sendbuf.
- * so pass it
- */
- if (conn->datadgst_en && ctask->sent - start > 0)
- iscsi_buf_data_digest_update(conn, &ctask->sendbuf);
-
- if (!ctask->data_count)
- break;
- iscsi_buf_init_sg(&ctask->sendbuf,
- &ctask->sg[ctask->sg_count++]);
- }
- BUG_ON(ctask->unsol_count < 0);
-
- /*
- * Done with the Data-Out. Next, check if we need
- * to send another unsolicited Data-Out.
- */
- if (ctask->unsol_count) {
- if (conn->datadgst_en) {
- if (iscsi_digest_final_send(conn, ctask,
- &dtask->digestbuf,
- &dtask->digest, 1)) {
- debug_tcp("send uns digest 0x%x fail\n",
- dtask->digest);
- return -EAGAIN;
- }
- debug_tcp("sending uns digest 0x%x, more uns\n",
- dtask->digest);
- }
- ctask->xmstate |= XMSTATE_UNS_INIT;
- return 1;
- }
-
- if (conn->datadgst_en && !(ctask->xmstate & XMSTATE_W_PAD)) {
- if (iscsi_digest_final_send(conn, ctask,
- &dtask->digestbuf,
- &dtask->digest, 1)) {
- debug_tcp("send last uns digest 0x%x fail\n",
- dtask->digest);
- return -EAGAIN;
- }
- debug_tcp("sending uns digest 0x%x\n",dtask->digest);
- }
-
- return 0;
-}
-
-static inline int
-handle_xmstate_sol_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
-{
- struct iscsi_session *session = conn->session;
- struct iscsi_r2t_info *r2t = ctask->r2t;
- struct iscsi_data_task *dtask = r2t->dtask;
- int left;
-
- ctask->xmstate &= ~XMSTATE_SOL_DATA;
- ctask->dtask = dtask;
-
- if (conn->datadgst_en) {
- iscsi_data_digest_init(conn, ctask);
- dtask->digest = 0;
- }
-solicit_again:
- /*
- * send Data-Out whitnin this R2T sequence.
- */
- if (!r2t->data_count)
- goto data_out_done;
-
- if (iscsi_sendpage(conn, &r2t->sendbuf, &r2t->data_count, &r2t->sent)) {
- ctask->xmstate |= XMSTATE_SOL_DATA;
- /* will continue with this ctask later.. */
- if (conn->datadgst_en) {
- crypto_digest_final(conn->data_tx_tfm,
- (u8 *)&dtask->digest);
- debug_tcp("r2t data send fail 0x%x\n", dtask->digest);
- }
- return -EAGAIN;
- }
-
- BUG_ON(r2t->data_count < 0);
- if (conn->datadgst_en)
- iscsi_buf_data_digest_update(conn, &r2t->sendbuf);
-
- if (r2t->data_count) {
- BUG_ON(ctask->sc->use_sg == 0);
- if (!iscsi_buf_left(&r2t->sendbuf)) {
- BUG_ON(ctask->bad_sg == r2t->sg);
- iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
- r2t->sg += 1;
- }
- goto solicit_again;
- }
-
-data_out_done:
- /*
- * Done with this Data-Out. Next, check if we have
- * to send another Data-Out for this R2T.
- */
- BUG_ON(r2t->data_length - r2t->sent < 0);
- left = r2t->data_length - r2t->sent;
- if (left) {
- if (conn->datadgst_en) {
- if (iscsi_digest_final_send(conn, ctask,
- &dtask->digestbuf,
- &dtask->digest, 1)) {
- debug_tcp("send r2t data digest 0x%x"
- "fail\n", dtask->digest);
- return -EAGAIN;
- }
- debug_tcp("r2t data send digest 0x%x\n",
- dtask->digest);
- }
- iscsi_solicit_data_cont(conn, ctask, r2t, left);
- ctask->xmstate |= XMSTATE_SOL_DATA;
- ctask->xmstate &= ~XMSTATE_SOL_HDR;
- return 1;
- }
-
- /*
- * Done with this R2T. Check if there are more
- * outstanding R2Ts ready to be processed.
- */
- BUG_ON(ctask->r2t_data_count - r2t->data_length < 0);
- if (conn->datadgst_en) {
- if (iscsi_digest_final_send(conn, ctask, &dtask->digestbuf,
- &dtask->digest, 1)) {
- debug_tcp("send last r2t data digest 0x%x"
- "fail\n", dtask->digest);
- return -EAGAIN;
- }
- debug_tcp("r2t done dout digest 0x%x\n", dtask->digest);
- }
-
- ctask->r2t_data_count -= r2t->data_length;
- ctask->r2t = NULL;
- spin_lock_bh(&session->lock);
- __kfifo_put(ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
- spin_unlock_bh(&session->lock);
- if (__kfifo_get(ctask->r2tqueue, (void*)&r2t, sizeof(void*))) {
- ctask->r2t = r2t;
- ctask->xmstate |= XMSTATE_SOL_DATA;
- ctask->xmstate &= ~XMSTATE_SOL_HDR;
- return 1;
- }
-
- return 0;
-}
-
-static inline int
-handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
-{
- struct iscsi_data_task *dtask = ctask->dtask;
- int sent;
-
- ctask->xmstate &= ~XMSTATE_W_PAD;
- iscsi_buf_init_virt(&ctask->sendbuf, (char*)&ctask->pad,
- ctask->pad_count);
- if (iscsi_sendpage(conn, &ctask->sendbuf, &ctask->pad_count, &sent)) {
- ctask->xmstate |= XMSTATE_W_PAD;
- return -EAGAIN;
- }
-
- if (conn->datadgst_en) {
- iscsi_buf_data_digest_update(conn, &ctask->sendbuf);
- /* imm data? */
- if (!dtask) {
- if (iscsi_digest_final_send(conn, ctask, &ctask->immbuf,
- &ctask->immdigest, 1)) {
- debug_tcp("send padding digest 0x%x"
- "fail!\n", ctask->immdigest);
- return -EAGAIN;
- }
- debug_tcp("done with padding, digest 0x%x\n",
- ctask->datadigest);
- } else {
- if (iscsi_digest_final_send(conn, ctask,
- &dtask->digestbuf,
- &dtask->digest, 1)) {
- debug_tcp("send padding digest 0x%x"
- "fail\n", dtask->digest);
- return -EAGAIN;
- }
- debug_tcp("done with padding, digest 0x%x\n",
- dtask->digest);
- }
- }
-
- return 0;
-}
-
-static int
-iscsi_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
-{
- int rc = 0;
-
- debug_scsi("ctask deq [cid %d xmstate %x itt 0x%x]\n",
- conn->id, ctask->xmstate, ctask->itt);
-
- /*
- * serialize with TMF AbortTask
- */
- if (ctask->mtask)
- return rc;
-
- if (ctask->xmstate & XMSTATE_R_HDR) {
- rc = handle_xmstate_r_hdr(conn, ctask);