drivers/target/target_core_transport.c

   1 /*******************************************************************************
   2  * Filename:  target_core_transport.c
   3  *
   4  * This file contains the Generic Target Engine Core.
   5  *
   6  * Copyright (c) 2002, 2003, 2004, 2005 PyX Technologies, Inc.
   7  * Copyright (c) 2005, 2006, 2007 SBE, Inc.
   8  * Copyright (c) 2007-2010 Rising Tide Systems
   9  * Copyright (c) 2008-2010 Linux-iSCSI.org
  10  *
  11  * Nicholas A. Bellinger <nab@kernel.org>
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of the GNU General Public License as published by
  15  * the Free Software Foundation; either version 2 of the License, or
  16  * (at your option) any later version.
  17  *
  18  * This program is distributed in the hope that it will be useful,
  19  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  20  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  21  * GNU General Public License for more details.
  22  *
  23  * You should have received a copy of the GNU General Public License
  24  * along with this program; if not, write to the Free Software
  25  * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
  26  *
  27  ******************************************************************************/
  28
  29 #include <linux/version.h>
  30 #include <linux/net.h>
  31 #include <linux/delay.h>
  32 #include <linux/string.h>
  33 #include <linux/timer.h>
  34 #include <linux/slab.h>
  35 #include <linux/blkdev.h>
  36 #include <linux/spinlock.h>
  37 #include <linux/kthread.h>
  38 #include <linux/in.h>
  39 #include <linux/cdrom.h>
  40 #include <asm/unaligned.h>
  41 #include <net/sock.h>
  42 #include <net/tcp.h>
  43 #include <scsi/scsi.h>
  44 #include <scsi/scsi_cmnd.h>
  45 #include <scsi/scsi_tcq.h>
  46
  47 #include <target/target_core_base.h>
  48 #include <target/target_core_device.h>
  49 #include <target/target_core_tmr.h>
  50 #include <target/target_core_tpg.h>
  51 #include <target/target_core_transport.h>
  52 #include <target/target_core_fabric_ops.h>
  53 #include <target/target_core_configfs.h>
  54
  55 #include "target_core_alua.h"
  56 #include "target_core_hba.h"
  57 #include "target_core_pr.h"
  58 #include "target_core_scdb.h"
  59 #include "target_core_ua.h"
  60
  61 /* #define DEBUG_CDB_HANDLER */
  62 #ifdef DEBUG_CDB_HANDLER
  63 #define DEBUG_CDB_H(x...) printk(KERN_INFO x)
  64 #else
  65 #define DEBUG_CDB_H(x...)
  66 #endif
  67
  68 /* #define DEBUG_CMD_MAP */
  69 #ifdef DEBUG_CMD_MAP
  70 #define DEBUG_CMD_M(x...) printk(KERN_INFO x)
  71 #else
  72 #define DEBUG_CMD_M(x...)
  73 #endif
  74
  75 /* #define DEBUG_MEM_ALLOC */
  76 #ifdef DEBUG_MEM_ALLOC
  77 #define DEBUG_MEM(x...) printk(KERN_INFO x)
  78 #else
  79 #define DEBUG_MEM(x...)
  80 #endif
  81
  82 /* #define DEBUG_MEM2_ALLOC */
  83 #ifdef DEBUG_MEM2_ALLOC
  84 #define DEBUG_MEM2(x...) printk(KERN_INFO x)
  85 #else
  86 #define DEBUG_MEM2(x...)
  87 #endif
  88
  89 /* #define DEBUG_SG_CALC */
  90 #ifdef DEBUG_SG_CALC
  91 #define DEBUG_SC(x...) printk(KERN_INFO x)
  92 #else
  93 #define DEBUG_SC(x...)
  94 #endif
  95
  96 /* #define DEBUG_SE_OBJ */
  97 #ifdef DEBUG_SE_OBJ
  98 #define DEBUG_SO(x...) printk(KERN_INFO x)
  99 #else
 100 #define DEBUG_SO(x...)
 101 #endif
 102
 103 /* #define DEBUG_CMD_VOL */
 104 #ifdef DEBUG_CMD_VOL
 105 #define DEBUG_VOL(x...) printk(KERN_INFO x)
 106 #else
 107 #define DEBUG_VOL(x...)
 108 #endif
 109
 110 /* #define DEBUG_CMD_STOP */
 111 #ifdef DEBUG_CMD_STOP
 112 #define DEBUG_CS(x...) printk(KERN_INFO x)
 113 #else
 114 #define DEBUG_CS(x...)
 115 #endif
 116
 117 /* #define DEBUG_PASSTHROUGH */
 118 #ifdef DEBUG_PASSTHROUGH
 119 #define DEBUG_PT(x...) printk(KERN_INFO x)
 120 #else
 121 #define DEBUG_PT(x...)
 122 #endif
 123
 124 /* #define DEBUG_TASK_STOP */
 125 #ifdef DEBUG_TASK_STOP
 126 #define DEBUG_TS(x...) printk(KERN_INFO x)
 127 #else
 128 #define DEBUG_TS(x...)
 129 #endif
 130
 131 /* #define DEBUG_TRANSPORT_STOP */
 132 #ifdef DEBUG_TRANSPORT_STOP
 133 #define DEBUG_TRANSPORT_S(x...) printk(KERN_INFO x)
 134 #else
 135 #define DEBUG_TRANSPORT_S(x...)
 136 #endif
 137
 138 /* #define DEBUG_TASK_FAILURE */
 139 #ifdef DEBUG_TASK_FAILURE
 140 #define DEBUG_TF(x...) printk(KERN_INFO x)
 141 #else
 142 #define DEBUG_TF(x...)
 143 #endif
 144
 145 /* #define DEBUG_DEV_OFFLINE */
 146 #ifdef DEBUG_DEV_OFFLINE
 147 #define DEBUG_DO(x...) printk(KERN_INFO x)
 148 #else
 149 #define DEBUG_DO(x...)
 150 #endif
 151
 152 /* #define DEBUG_TASK_STATE */
 153 #ifdef DEBUG_TASK_STATE
 154 #define DEBUG_TSTATE(x...) printk(KERN_INFO x)
 155 #else
 156 #define DEBUG_TSTATE(x...)
 157 #endif
 158
 159 /* #define DEBUG_STATUS_THR */
 160 #ifdef DEBUG_STATUS_THR
 161 #define DEBUG_ST(x...) printk(KERN_INFO x)
 162 #else
 163 #define DEBUG_ST(x...)
 164 #endif
 165
 166 /* #define DEBUG_TASK_TIMEOUT */
 167 #ifdef DEBUG_TASK_TIMEOUT
 168 #define DEBUG_TT(x...) printk(KERN_INFO x)
 169 #else
 170 #define DEBUG_TT(x...)
 171 #endif
 172
 173 /* #define DEBUG_GENERIC_REQUEST_FAILURE */
 174 #ifdef DEBUG_GENERIC_REQUEST_FAILURE
 175 #define DEBUG_GRF(x...) printk(KERN_INFO x)
 176 #else
 177 #define DEBUG_GRF(x...)
 178 #endif
 179
 180 /* #define DEBUG_SAM_TASK_ATTRS */
 181 #ifdef DEBUG_SAM_TASK_ATTRS
 182 #define DEBUG_STA(x...) printk(KERN_INFO x)
 183 #else
 184 #define DEBUG_STA(x...)
 185 #endif
 186
 187 static int sub_api_initialized;
 188
 189 static struct kmem_cache *se_cmd_cache;
 190 static struct kmem_cache *se_sess_cache;
 191 struct kmem_cache *se_tmr_req_cache;
 192 struct kmem_cache *se_ua_cache;
 193 struct kmem_cache *se_mem_cache;
 194 struct kmem_cache *t10_pr_reg_cache;
 195 struct kmem_cache *t10_alua_lu_gp_cache;
 196 struct kmem_cache *t10_alua_lu_gp_mem_cache;
 197 struct kmem_cache *t10_alua_tg_pt_gp_cache;
 198 struct kmem_cache *t10_alua_tg_pt_gp_mem_cache;
 199
 200 /* Used for transport_dev_get_map_*() */
 201 typedef int (*map_func_t)(struct se_task *, u32);
 202
 203 static int transport_generic_write_pending(struct se_cmd *);
 204 static int transport_processing_thread(void *param);
 205 static int __transport_execute_tasks(struct se_device *dev);
 206 static void transport_complete_task_attr(struct se_cmd *cmd);
 207 static void transport_direct_request_timeout(struct se_cmd *cmd);
 208 static void transport_free_dev_tasks(struct se_cmd *cmd);
 209 static u32 transport_generic_get_cdb_count(struct se_cmd *cmd,
 210                 unsigned long long starting_lba, u32 sectors,
 211                 enum dma_data_direction data_direction,
 212                 struct list_head *mem_list, int set_counts);
 213 static int transport_generic_get_mem(struct se_cmd *cmd, u32 length,
 214                 u32 dma_size);
 215 static int transport_generic_remove(struct se_cmd *cmd,
 216                 int release_to_pool, int session_reinstatement);
 217 static int transport_get_sectors(struct se_cmd *cmd);
 218 static int transport_map_sg_to_mem(struct se_cmd *cmd,
 219                 struct list_head *se_mem_list, struct scatterlist *sgl,
 220                 u32 *se_mem_cnt);
 221 static void transport_memcpy_se_mem_read_contig(struct se_cmd *cmd,
 222                 unsigned char *dst, struct list_head *se_mem_list);
 223 static void transport_release_fe_cmd(struct se_cmd *cmd);
 224 static void transport_remove_cmd_from_queue(struct se_cmd *cmd,
 225                 struct se_queue_obj *qobj);
 226 static int transport_set_sense_codes(struct se_cmd *cmd, u8 asc, u8 ascq);
 227 static void transport_stop_all_task_timers(struct se_cmd *cmd);
 228
 229 int init_se_kmem_caches(void)
 230 {
 231         se_cmd_cache = kmem_cache_create("se_cmd_cache",
 232                         sizeof(struct se_cmd), __alignof__(struct se_cmd), 0, NULL);
 233         if (!(se_cmd_cache)) {
 234                 printk(KERN_ERR "kmem_cache_create for struct se_cmd failed\n");
 235                 goto out;
 236         }
 237         se_tmr_req_cache = kmem_cache_create("se_tmr_cache",
 238                         sizeof(struct se_tmr_req), __alignof__(struct se_tmr_req),
 239                         0, NULL);
 240         if (!(se_tmr_req_cache)) {
 241                 printk(KERN_ERR "kmem_cache_create() for struct se_tmr_req"
 242                                 " failed\n");
 243                 goto out;
 244         }
 245         se_sess_cache = kmem_cache_create("se_sess_cache",
 246                         sizeof(struct se_session), __alignof__(struct se_session),
 247                         0, NULL);
 248         if (!(se_sess_cache)) {
 249                 printk(KERN_ERR "kmem_cache_create() for struct se_session"
 250                                 " failed\n");
 251                 goto out;
 252         }
 253         se_ua_cache = kmem_cache_create("se_ua_cache",
 254                         sizeof(struct se_ua), __alignof__(struct se_ua),
 255                         0, NULL);
 256         if (!(se_ua_cache)) {
 257                 printk(KERN_ERR "kmem_cache_create() for struct se_ua failed\n");
 258                 goto out;
 259         }
 260         se_mem_cache = kmem_cache_create("se_mem_cache",
 261                         sizeof(struct se_mem), __alignof__(struct se_mem), 0, NULL);
 262         if (!(se_mem_cache)) {
 263                 printk(KERN_ERR "kmem_cache_create() for struct se_mem failed\n");
 264                 goto out;
 265         }
 266         t10_pr_reg_cache = kmem_cache_create("t10_pr_reg_cache",
 267                         sizeof(struct t10_pr_registration),
 268                         __alignof__(struct t10_pr_registration), 0, NULL);
 269         if (!(t10_pr_reg_cache)) {
 270                 printk(KERN_ERR "kmem_cache_create() for struct t10_pr_registration"
 271                                 " failed\n");
 272                 goto out;
 273         }
 274         t10_alua_lu_gp_cache = kmem_cache_create("t10_alua_lu_gp_cache",
 275                         sizeof(struct t10_alua_lu_gp), __alignof__(struct t10_alua_lu_gp),
 276                         0, NULL);
 277         if (!(t10_alua_lu_gp_cache)) {
 278                 printk(KERN_ERR "kmem_cache_create() for t10_alua_lu_gp_cache"
 279                                 " failed\n");
 280                 goto out;
 281         }
 282         t10_alua_lu_gp_mem_cache = kmem_cache_create("t10_alua_lu_gp_mem_cache",
 283                         sizeof(struct t10_alua_lu_gp_member),
 284                         __alignof__(struct t10_alua_lu_gp_member), 0, NULL);
 285         if (!(t10_alua_lu_gp_mem_cache)) {
 286                 printk(KERN_ERR "kmem_cache_create() for t10_alua_lu_gp_mem_"
 287                                 "cache failed\n");
 288                 goto out;
 289         }
 290         t10_alua_tg_pt_gp_cache = kmem_cache_create("t10_alua_tg_pt_gp_cache",
 291                         sizeof(struct t10_alua_tg_pt_gp),
 292                         __alignof__(struct t10_alua_tg_pt_gp), 0, NULL);
 293         if (!(t10_alua_tg_pt_gp_cache)) {
 294                 printk(KERN_ERR "kmem_cache_create() for t10_alua_tg_pt_gp_"
 295                                 "cache failed\n");
 296                 goto out;
 297         }
 298         t10_alua_tg_pt_gp_mem_cache = kmem_cache_create(
 299                         "t10_alua_tg_pt_gp_mem_cache",
 300                         sizeof(struct t10_alua_tg_pt_gp_member),
 301                         __alignof__(struct t10_alua_tg_pt_gp_member),
 302                         0, NULL);
 303         if (!(t10_alua_tg_pt_gp_mem_cache)) {
 304                 printk(KERN_ERR "kmem_cache_create() for t10_alua_tg_pt_gp_"
 305                                 "mem_t failed\n");
 306                 goto out;
 307         }
 308
 309         return 0;
 310 out:
 311         if (se_cmd_cache)
 312                 kmem_cache_destroy(se_cmd_cache);
 313         if (se_tmr_req_cache)
 314                 kmem_cache_destroy(se_tmr_req_cache);
 315         if (se_sess_cache)
 316                 kmem_cache_destroy(se_sess_cache);
 317         if (se_ua_cache)
 318                 kmem_cache_destroy(se_ua_cache);
 319         if (se_mem_cache)
 320                 kmem_cache_destroy(se_mem_cache);
 321         if (t10_pr_reg_cache)
 322                 kmem_cache_destroy(t10_pr_reg_cache);
 323         if (t10_alua_lu_gp_cache)
 324                 kmem_cache_destroy(t10_alua_lu_gp_cache);
 325         if (t10_alua_lu_gp_mem_cache)
 326                 kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
 327         if (t10_alua_tg_pt_gp_cache)
 328                 kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
 329         if (t10_alua_tg_pt_gp_mem_cache)
 330                 kmem_cache_destroy(t10_alua_tg_pt_gp_mem_cache);
 331         return -ENOMEM;
 332 }
 333
 334 void release_se_kmem_caches(void)
 335 {
 336         kmem_cache_destroy(se_cmd_cache);
 337         kmem_cache_destroy(se_tmr_req_cache);
 338         kmem_cache_destroy(se_sess_cache);
 339         kmem_cache_destroy(se_ua_cache);
 340         kmem_cache_destroy(se_mem_cache);
 341         kmem_cache_destroy(t10_pr_reg_cache);
 342         kmem_cache_destroy(t10_alua_lu_gp_cache);
 343         kmem_cache_destroy(t10_alua_lu_gp_mem_cache);
 344         kmem_cache_destroy(t10_alua_tg_pt_gp_cache);
 345         kmem_cache_destroy(t10_alua_tg_pt_gp_mem_cache);
 346 }
 347
 348 /* This code ensures unique mib indexes are handed out. */
 349 static DEFINE_SPINLOCK(scsi_mib_index_lock);
 350 static u32 scsi_mib_index[SCSI_INDEX_TYPE_MAX];
 351
 352 /*
 353  * Allocate a new row index for the entry type specified
 354  */
 355 u32 scsi_get_new_index(scsi_index_t type)
 356 {
 357         u32 new_index;
 358
 359         BUG_ON((type < 0) || (type >= SCSI_INDEX_TYPE_MAX));
 360
 361         spin_lock(&scsi_mib_index_lock);
 362         new_index = ++scsi_mib_index[type];
 363         spin_unlock(&scsi_mib_index_lock);
 364
 365         return new_index;
 366 }
 367
 368 void transport_init_queue_obj(struct se_queue_obj *qobj)
 369 {
 370         atomic_set(&qobj->queue_cnt, 0);
 371         INIT_LIST_HEAD(&qobj->qobj_list);
 372         init_waitqueue_head(&qobj->thread_wq);
 373         spin_lock_init(&qobj->cmd_queue_lock);
 374 }
 375 EXPORT_SYMBOL(transport_init_queue_obj);
 376
 377 static int transport_subsystem_reqmods(void)
 378 {
 379         int ret;
 380
 381         ret = request_module("target_core_iblock");
 382         if (ret != 0)
 383                 printk(KERN_ERR "Unable to load target_core_iblock\n");
 384
 385         ret = request_module("target_core_file");
 386         if (ret != 0)
 387                 printk(KERN_ERR "Unable to load target_core_file\n");
 388
 389         ret = request_module("target_core_pscsi");
 390         if (ret != 0)
 391                 printk(KERN_ERR "Unable to load target_core_pscsi\n");
 392
 393         ret = request_module("target_core_stgt");
 394         if (ret != 0)
 395                 printk(KERN_ERR "Unable to load target_core_stgt\n");
 396
 397         return 0;
 398 }
 399
 400 int transport_subsystem_check_init(void)
 401 {
 402         int ret;
 403
 404         if (sub_api_initialized)
 405                 return 0;
 406         /*
 407          * Request the loading of known TCM subsystem plugins..
 408          */
 409         ret = transport_subsystem_reqmods();
 410         if (ret < 0)
 411                 return ret;
 412
 413         sub_api_initialized = 1;
 414         return 0;
 415 }
 416
 417 struct se_session *transport_init_session(void)
 418 {
 419         struct se_session *se_sess;
 420
 421         se_sess = kmem_cache_zalloc(se_sess_cache, GFP_KERNEL);
 422         if (!(se_sess)) {
 423                 printk(KERN_ERR "Unable to allocate struct se_session from"
 424                                 " se_sess_cache\n");
 425                 return ERR_PTR(-ENOMEM);
 426         }
 427         INIT_LIST_HEAD(&se_sess->sess_list);
 428         INIT_LIST_HEAD(&se_sess->sess_acl_list);
 429
 430         return se_sess;
 431 }
 432 EXPORT_SYMBOL(transport_init_session);
 433
 434 /*
 435  * Called with spin_lock_bh(&struct se_portal_group->session_lock called.
 436  */
 437 void __transport_register_session(
 438         struct se_portal_group *se_tpg,
 439         struct se_node_acl *se_nacl,
 440         struct se_session *se_sess,
 441         void *fabric_sess_ptr)
 442 {
 443         unsigned char buf[PR_REG_ISID_LEN];
 444
 445         se_sess->se_tpg = se_tpg;
 446         se_sess->fabric_sess_ptr = fabric_sess_ptr;
 447         /*
 448          * Used by struct se_node_acl's under ConfigFS to locate active se_session-t
 449          *
 450          * Only set for struct se_session's that will actually be moving I/O.
 451          * eg: *NOT* discovery sessions.
 452          */
 453         if (se_nacl) {
 454                 /*
 455                  * If the fabric module supports an ISID based TransportID,
 456                  * save this value in binary from the fabric I_T Nexus now.
 457                  */
 458                 if (se_tpg->se_tpg_tfo->sess_get_initiator_sid != NULL) {
 459                         memset(&buf[0], 0, PR_REG_ISID_LEN);
 460                         se_tpg->se_tpg_tfo->sess_get_initiator_sid(se_sess,
 461                                         &buf[0], PR_REG_ISID_LEN);
 462                         se_sess->sess_bin_isid = get_unaligned_be64(&buf[0]);
 463                 }
 464                 spin_lock_irq(&se_nacl->nacl_sess_lock);
 465                 /*
 466                  * The se_nacl->nacl_sess pointer will be set to the
 467                  * last active I_T Nexus for each struct se_node_acl.
 468                  */
 469                 se_nacl->nacl_sess = se_sess;
 470
 471                 list_add_tail(&se_sess->sess_acl_list,
 472                               &se_nacl->acl_sess_list);
 473                 spin_unlock_irq(&se_nacl->nacl_sess_lock);
 474         }
 475         list_add_tail(&se_sess->sess_list, &se_tpg->tpg_sess_list);
 476
 477         printk(KERN_INFO "TARGET_CORE[%s]: Registered fabric_sess_ptr: %p\n",
 478                 se_tpg->se_tpg_tfo->get_fabric_name(), se_sess->fabric_sess_ptr);
 479 }
 480 EXPORT_SYMBOL(__transport_register_session);
 481
 482 void transport_register_session(
 483         struct se_portal_group *se_tpg,
 484         struct se_node_acl *se_nacl,
 485         struct se_session *se_sess,
 486         void *fabric_sess_ptr)
 487 {
 488         spin_lock_bh(&se_tpg->session_lock);
 489         __transport_register_session(se_tpg, se_nacl, se_sess, fabric_sess_ptr);
 490         spin_unlock_bh(&se_tpg->session_lock);
 491 }
 492 EXPORT_SYMBOL(transport_register_session);
 493
 494 void transport_deregister_session_configfs(struct se_session *se_sess)
 495 {
 496         struct se_node_acl *se_nacl;
 497         unsigned long flags;
 498         /*
 499          * Used by struct se_node_acl's under ConfigFS to locate active struct se_session
 500          */
 501         se_nacl = se_sess->se_node_acl;
 502         if ((se_nacl)) {
 503                 spin_lock_irqsave(&se_nacl->nacl_sess_lock, flags);
 504                 list_del(&se_sess->sess_acl_list);
 505                 /*
 506                  * If the session list is empty, then clear the pointer.
 507                  * Otherwise, set the struct se_session pointer from the tail
 508                  * element of the per struct se_node_acl active session list.
 509                  */
 510                 if (list_empty(&se_nacl->acl_sess_list))
 511                         se_nacl->nacl_sess = NULL;
 512                 else {
 513                         se_nacl->nacl_sess = container_of(
 514                                         se_nacl->acl_sess_list.prev,
 515                                         struct se_session, sess_acl_list);
 516                 }
 517                 spin_unlock_irqrestore(&se_nacl->nacl_sess_lock, flags);
 518         }
 519 }
 520 EXPORT_SYMBOL(transport_deregister_session_configfs);
 521
 522 void transport_free_session(struct se_session *se_sess)
 523 {
 524         kmem_cache_free(se_sess_cache, se_sess);
 525 }
 526 EXPORT_SYMBOL(transport_free_session);
 527
 528 void transport_deregister_session(struct se_session *se_sess)
 529 {
 530         struct se_portal_group *se_tpg = se_sess->se_tpg;
 531         struct se_node_acl *se_nacl;
 532
 533         if (!(se_tpg)) {
 534                 transport_free_session(se_sess);
 535                 return;
 536         }
 537
 538         spin_lock_bh(&se_tpg->session_lock);
 539         list_del(&se_sess->sess_list);
 540         se_sess->se_tpg = NULL;
 541         se_sess->fabric_sess_ptr = NULL;
 542         spin_unlock_bh(&se_tpg->session_lock);
 543
 544         /*
 545          * Determine if we need to do extra work for this initiator node's
 546          * struct se_node_acl if it had been previously dynamically generated.
 547          */
 548         se_nacl = se_sess->se_node_acl;
 549         if ((se_nacl)) {
 550                 spin_lock_bh(&se_tpg->acl_node_lock);
 551                 if (se_nacl->dynamic_node_acl) {
 552                         if (!(se_tpg->se_tpg_tfo->tpg_check_demo_mode_cache(
 553                                         se_tpg))) {
 554                                 list_del(&se_nacl->acl_list);
 555                                 se_tpg->num_node_acls--;
 556                                 spin_unlock_bh(&se_tpg->acl_node_lock);
 557
 558                                 core_tpg_wait_for_nacl_pr_ref(se_nacl);
 559                                 core_free_device_list_for_node(se_nacl, se_tpg);
 560                                 se_tpg->se_tpg_tfo->tpg_release_fabric_acl(se_tpg,
 561                                                 se_nacl);
 562                                 spin_lock_bh(&se_tpg->acl_node_lock);
 563                         }
 564                 }
 565                 spin_unlock_bh(&se_tpg->acl_node_lock);
 566         }
 567
 568         transport_free_session(se_sess);
 569
 570         printk(KERN_INFO "TARGET_CORE[%s]: Deregistered fabric_sess\n",
 571                 se_tpg->se_tpg_tfo->get_fabric_name());
 572 }
 573 EXPORT_SYMBOL(transport_deregister_session);
 574
 575 /*
 576  * Called with cmd->t_task.t_state_lock held.
 577  */
 578 static void transport_all_task_dev_remove_state(struct se_cmd *cmd)
 579 {
 580         struct se_device *dev;
 581         struct se_task *task;
 582         unsigned long flags;
 583
 584         list_for_each_entry(task, &cmd->t_task.t_task_list, t_list) {
 585                 dev = task->se_dev;
 586                 if (!(dev))
 587                         continue;
 588
 589                 if (atomic_read(&task->task_active))
 590                         continue;
 591
 592                 if (!(atomic_read(&task->task_state_active)))
 593                         continue;
 594
 595                 spin_lock_irqsave(&dev->execute_task_lock, flags);
 596                 list_del(&task->t_state_list);
 597                 DEBUG_TSTATE("Removed ITT: 0x%08x dev: %p task[%p]\n",
 598                         cmd->se_tfo->tfo_get_task_tag(cmd), dev, task);
 599                 spin_unlock_irqrestore(&dev->execute_task_lock, flags);
 600
 601                 atomic_set(&task->task_state_active, 0);
 602                 atomic_dec(&cmd->t_task.t_task_cdbs_ex_left);
 603         }
 604 }
 605
 606 /*      transport_cmd_check_stop():
 607  *
 608  *      'transport_off = 1' determines if t_transport_active should be cleared.
 609  *      'transport_off = 2' determines if task_dev_state should be removed.
 610  *
 611  *      A non-zero u8 t_state sets cmd->t_state.
 612  *      Returns 1 when command is stopped, else 0.
 613  */
 614 static int transport_cmd_check_stop(
 615         struct se_cmd *cmd,
 616         int transport_off,
 617         u8 t_state)
 618 {
 619         unsigned long flags;
 620
 621         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
 622         /*
 623          * Determine if IOCTL context caller in requesting the stopping of this
 624          * command for LUN shutdown purposes.
 625          */
 626         if (atomic_read(&cmd->t_task.transport_lun_stop)) {
 627                 DEBUG_CS("%s:%d atomic_read(&cmd->t_task.transport_lun_stop)"
 628                         " == TRUE for ITT: 0x%08x\n", __func__, __LINE__,
 629                         cmd->se_tfo->get_task_tag(cmd));
 630
 631                 cmd->deferred_t_state = cmd->t_state;
 632                 cmd->t_state = TRANSPORT_DEFERRED_CMD;
 633                 atomic_set(&cmd->t_task.t_transport_active, 0);
 634                 if (transport_off == 2)
 635                         transport_all_task_dev_remove_state(cmd);
 636                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
 637
 638                 complete(&cmd->t_task.transport_lun_stop_comp);
 639                 return 1;
 640         }
 641         /*
 642          * Determine if frontend context caller is requesting the stopping of
 643          * this command for frontend exceptions.
 644          */
 645         if (atomic_read(&cmd->t_task.t_transport_stop)) {
 646                 DEBUG_CS("%s:%d atomic_read(&cmd->t_task.t_transport_stop) =="
 647                         " TRUE for ITT: 0x%08x\n", __func__, __LINE__,
 648                         cmd->se_tfo->get_task_tag(cmd));
 649
 650                 cmd->deferred_t_state = cmd->t_state;
 651                 cmd->t_state = TRANSPORT_DEFERRED_CMD;
 652                 if (transport_off == 2)
 653                         transport_all_task_dev_remove_state(cmd);
 654
 655                 /*
 656                  * Clear struct se_cmd->se_lun before the transport_off == 2 handoff
 657                  * to FE.
 658                  */
 659                 if (transport_off == 2)
 660                         cmd->se_lun = NULL;
 661                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
 662
 663                 complete(&cmd->t_task.t_transport_stop_comp);
 664                 return 1;
 665         }
 666         if (transport_off) {
 667                 atomic_set(&cmd->t_task.t_transport_active, 0);
 668                 if (transport_off == 2) {
 669                         transport_all_task_dev_remove_state(cmd);
 670                         /*
 671                          * Clear struct se_cmd->se_lun before the transport_off == 2
 672                          * handoff to fabric module.
 673                          */
 674                         cmd->se_lun = NULL;
 675                         /*
 676                          * Some fabric modules like tcm_loop can release
 677                          * their internally allocated I/O reference now and
 678                          * struct se_cmd now.
 679                          */
 680                         if (cmd->se_tfo->check_stop_free != NULL) {
 681                                 spin_unlock_irqrestore(
 682                                         &cmd->t_task.t_state_lock, flags);
 683
 684                                 cmd->se_tfo->check_stop_free(cmd);
 685                                 return 1;
 686                         }
 687                 }
 688                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
 689
 690                 return 0;
 691         } else if (t_state)
 692                 cmd->t_state = t_state;
 693         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
 694
 695         return 0;
 696 }
 697
 698 static int transport_cmd_check_stop_to_fabric(struct se_cmd *cmd)
 699 {
 700         return transport_cmd_check_stop(cmd, 2, 0);
 701 }
 702
 703 static void transport_lun_remove_cmd(struct se_cmd *cmd)
 704 {
 705         struct se_lun *lun = cmd->se_lun;
 706         unsigned long flags;
 707
 708         if (!lun)
 709                 return;
 710
 711         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
 712         if (!(atomic_read(&cmd->t_task.transport_dev_active))) {
 713                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
 714                 goto check_lun;
 715         }
 716         atomic_set(&cmd->t_task.transport_dev_active, 0);
 717         transport_all_task_dev_remove_state(cmd);
 718         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
 719
 720
 721 check_lun:
 722         spin_lock_irqsave(&lun->lun_cmd_lock, flags);
 723         if (atomic_read(&cmd->t_task.transport_lun_active)) {
 724                 list_del(&cmd->se_lun_node);
 725                 atomic_set(&cmd->t_task.transport_lun_active, 0);
 726 #if 0
 727                 printk(KERN_INFO "Removed ITT: 0x%08x from LUN LIST[%d]\n"
 728                         cmd->se_tfo->get_task_tag(cmd), lun->unpacked_lun);
 729 #endif
 730         }
 731         spin_unlock_irqrestore(&lun->lun_cmd_lock, flags);
 732 }
 733
 734 void transport_cmd_finish_abort(struct se_cmd *cmd, int remove)
 735 {
 736         transport_remove_cmd_from_queue(cmd, &cmd->se_dev->dev_queue_obj);
 737         transport_lun_remove_cmd(cmd);
 738
 739         if (transport_cmd_check_stop_to_fabric(cmd))
 740                 return;
 741         if (remove)
 742                 transport_generic_remove(cmd, 0, 0);
 743 }
 744
 745 void transport_cmd_finish_abort_tmr(struct se_cmd *cmd)
 746 {
 747         transport_remove_cmd_from_queue(cmd, &cmd->se_dev->dev_queue_obj);
 748
 749         if (transport_cmd_check_stop_to_fabric(cmd))
 750                 return;
 751
 752         transport_generic_remove(cmd, 0, 0);
 753 }
 754
 755 static void transport_add_cmd_to_queue(
 756         struct se_cmd *cmd,
 757         int t_state)
 758 {
 759         struct se_device *dev = cmd->se_dev;
 760         struct se_queue_obj *qobj = &dev->dev_queue_obj;
 761         unsigned long flags;
 762
 763         INIT_LIST_HEAD(&cmd->se_queue_node);
 764
 765         if (t_state) {
 766                 spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
 767                 cmd->t_state = t_state;
 768                 atomic_set(&cmd->t_task.t_transport_active, 1);
 769                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
 770         }
 771
 772         spin_lock_irqsave(&qobj->cmd_queue_lock, flags);
 773         list_add_tail(&cmd->se_queue_node, &qobj->qobj_list);
 774         atomic_inc(&cmd->t_task.t_transport_queue_active);
 775         spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
 776
 777         atomic_inc(&qobj->queue_cnt);
 778         wake_up_interruptible(&qobj->thread_wq);
 779 }
 780
 781 static struct se_cmd *
 782 transport_get_cmd_from_queue(struct se_queue_obj *qobj)
 783 {
 784         struct se_cmd *cmd;
 785         unsigned long flags;
 786
 787         spin_lock_irqsave(&qobj->cmd_queue_lock, flags);
 788         if (list_empty(&qobj->qobj_list)) {
 789                 spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
 790                 return NULL;
 791         }
 792         cmd = list_first_entry(&qobj->qobj_list, struct se_cmd, se_queue_node);
 793
 794         atomic_dec(&cmd->t_task.t_transport_queue_active);
 795
 796         list_del(&cmd->se_queue_node);
 797         atomic_dec(&qobj->queue_cnt);
 798         spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
 799
 800         return cmd;
 801 }
 802
 803 static void transport_remove_cmd_from_queue(struct se_cmd *cmd,
 804                 struct se_queue_obj *qobj)
 805 {
 806         struct se_cmd *t;
 807         unsigned long flags;
 808
 809         spin_lock_irqsave(&qobj->cmd_queue_lock, flags);
 810         if (!(atomic_read(&cmd->t_task.t_transport_queue_active))) {
 811                 spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
 812                 return;
 813         }
 814
 815         list_for_each_entry(t, &qobj->qobj_list, se_queue_node)
 816                 if (t == cmd) {
 817                         atomic_dec(&cmd->t_task.t_transport_queue_active);
 818                         atomic_dec(&qobj->queue_cnt);
 819                         list_del(&cmd->se_queue_node);
 820                         break;
 821                 }
 822         spin_unlock_irqrestore(&qobj->cmd_queue_lock, flags);
 823
 824         if (atomic_read(&cmd->t_task.t_transport_queue_active)) {
 825                 printk(KERN_ERR "ITT: 0x%08x t_transport_queue_active: %d\n",
 826                         cmd->se_tfo->get_task_tag(cmd),
 827                         atomic_read(&cmd->t_task.t_transport_queue_active));
 828         }
 829 }
 830
 831 /*
 832  * Completion function used by TCM subsystem plugins (such as FILEIO)
 833  * for queueing up response from struct se_subsystem_api->do_task()
 834  */
 835 void transport_complete_sync_cache(struct se_cmd *cmd, int good)
 836 {
 837         struct se_task *task = list_entry(cmd->t_task.t_task_list.next,
 838                                 struct se_task, t_list);
 839
 840         if (good) {
 841                 cmd->scsi_status = SAM_STAT_GOOD;
 842                 task->task_scsi_status = GOOD;
 843         } else {
 844                 task->task_scsi_status = SAM_STAT_CHECK_CONDITION;
 845                 task->task_error_status = PYX_TRANSPORT_ILLEGAL_REQUEST;
 846                 task->task_se_cmd->transport_error_status =
 847                                         PYX_TRANSPORT_ILLEGAL_REQUEST;
 848         }
 849
 850         transport_complete_task(task, good);
 851 }
 852 EXPORT_SYMBOL(transport_complete_sync_cache);
 853
 854 /*      transport_complete_task():
 855  *
 856  *      Called from interrupt and non interrupt context depending
 857  *      on the transport plugin.
 858  */
 859 void transport_complete_task(struct se_task *task, int success)
 860 {
 861         struct se_cmd *cmd = task->task_se_cmd;
 862         struct se_device *dev = task->se_dev;
 863         int t_state;
 864         unsigned long flags;
 865 #if 0
 866         printk(KERN_INFO "task: %p CDB: 0x%02x obj_ptr: %p\n", task,
 867                         cmd->t_task.t_task_cdb[0], dev);
 868 #endif
 869         if (dev)
 870                 atomic_inc(&dev->depth_left);
 871
 872         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
 873         atomic_set(&task->task_active, 0);
 874
 875         /*
 876          * See if any sense data exists, if so set the TASK_SENSE flag.
 877          * Also check for any other post completion work that needs to be
 878          * done by the plugins.
 879          */
 880         if (dev && dev->transport->transport_complete) {
 881                 if (dev->transport->transport_complete(task) != 0) {
 882                         cmd->se_cmd_flags |= SCF_TRANSPORT_TASK_SENSE;
 883                         task->task_sense = 1;
 884                         success = 1;
 885                 }
 886         }
 887
 888         /*
 889          * See if we are waiting for outstanding struct se_task
 890          * to complete for an exception condition
 891          */
 892         if (atomic_read(&task->task_stop)) {
 893                 /*
 894                  * Decrement cmd->t_task.t_se_count if this task had
 895                  * previously thrown its timeout exception handler.
 896                  */
 897                 if (atomic_read(&task->task_timeout)) {
 898                         atomic_dec(&cmd->t_task.t_se_count);
 899                         atomic_set(&task->task_timeout, 0);
 900                 }
 901                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
 902
 903                 complete(&task->task_stop_comp);
 904                 return;
 905         }
 906         /*
 907          * If the task's timeout handler has fired, use the t_task_cdbs_timeout
 908          * left counter to determine when the struct se_cmd is ready to be queued to
 909          * the processing thread.
 910          */
 911         if (atomic_read(&task->task_timeout)) {
 912                 if (!(atomic_dec_and_test(
 913                                 &cmd->t_task.t_task_cdbs_timeout_left))) {
 914                         spin_unlock_irqrestore(&cmd->t_task.t_state_lock,
 915                                 flags);
 916                         return;
 917                 }
 918                 t_state = TRANSPORT_COMPLETE_TIMEOUT;
 919                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
 920
 921                 transport_add_cmd_to_queue(cmd, t_state);
 922                 return;
 923         }
 924         atomic_dec(&cmd->t_task.t_task_cdbs_timeout_left);
 925
 926         /*
 927          * Decrement the outstanding t_task_cdbs_left count.  The last
 928          * struct se_task from struct se_cmd will complete itself into the
 929          * device queue depending upon int success.
 930          */
 931         if (!(atomic_dec_and_test(&cmd->t_task.t_task_cdbs_left))) {
 932                 if (!success)
 933                         cmd->t_task.t_tasks_failed = 1;
 934
 935                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
 936                 return;
 937         }
 938
 939         if (!success || cmd->t_task.t_tasks_failed) {
 940                 t_state = TRANSPORT_COMPLETE_FAILURE;
 941                 if (!task->task_error_status) {
 942                         task->task_error_status =
 943                                 PYX_TRANSPORT_UNKNOWN_SAM_OPCODE;
 944                         cmd->transport_error_status =
 945                                 PYX_TRANSPORT_UNKNOWN_SAM_OPCODE;
 946                 }
 947         } else {
 948                 atomic_set(&cmd->t_task.t_transport_complete, 1);
 949                 t_state = TRANSPORT_COMPLETE_OK;
 950         }
 951         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
 952
 953         transport_add_cmd_to_queue(cmd, t_state);
 954 }
 955 EXPORT_SYMBOL(transport_complete_task);
 956
 957 /*
 958  * Called by transport_add_tasks_from_cmd() once a struct se_cmd's
 959  * struct se_task list are ready to be added to the active execution list
 960  * struct se_device
 961
 962  * Called with se_dev_t->execute_task_lock called.
 963  */
 964 static inline int transport_add_task_check_sam_attr(
 965         struct se_task *task,
 966         struct se_task *task_prev,
 967         struct se_device *dev)
 968 {
 969         /*
 970          * No SAM Task attribute emulation enabled, add to tail of
 971          * execution queue
 972          */
 973         if (dev->dev_task_attr_type != SAM_TASK_ATTR_EMULATED) {
 974                 list_add_tail(&task->t_execute_list, &dev->execute_task_list);
 975                 return 0;
 976         }
 977         /*
 978          * HEAD_OF_QUEUE attribute for received CDB, which means
 979          * the first task that is associated with a struct se_cmd goes to
 980          * head of the struct se_device->execute_task_list, and task_prev
 981          * after that for each subsequent task
 982          */
 983         if (task->task_se_cmd->sam_task_attr == MSG_HEAD_TAG) {
 984                 list_add(&task->t_execute_list,
 985                                 (task_prev != NULL) ?
 986                                 &task_prev->t_execute_list :
 987                                 &dev->execute_task_list);
 988
 989                 DEBUG_STA("Set HEAD_OF_QUEUE for task CDB: 0x%02x"
 990                                 " in execution queue\n",
 991                                 T_TASK(task->task_se_cmd)->t_task_cdb[0]);
 992                 return 1;
 993         }
 994         /*
 995          * For ORDERED, SIMPLE or UNTAGGED attribute tasks once they have been
 996          * transitioned from Dermant -> Active state, and are added to the end
 997          * of the struct se_device->execute_task_list
 998          */
 999         list_add_tail(&task->t_execute_list, &dev->execute_task_list);
1000         return 0;
1001 }
1002
1003 /*      __transport_add_task_to_execute_queue():
1004  *
1005  *      Called with se_dev_t->execute_task_lock called.
1006  */
1007 static void __transport_add_task_to_execute_queue(
1008         struct se_task *task,
1009         struct se_task *task_prev,
1010         struct se_device *dev)
1011 {
1012         int head_of_queue;
1013
1014         head_of_queue = transport_add_task_check_sam_attr(task, task_prev, dev);
1015         atomic_inc(&dev->execute_tasks);
1016
1017         if (atomic_read(&task->task_state_active))
1018                 return;
1019         /*
1020          * Determine if this task needs to go to HEAD_OF_QUEUE for the
1021          * state list as well.  Running with SAM Task Attribute emulation
1022          * will always return head_of_queue == 0 here
1023          */
1024         if (head_of_queue)
1025                 list_add(&task->t_state_list, (task_prev) ?
1026                                 &task_prev->t_state_list :
1027                                 &dev->state_task_list);
1028         else
1029                 list_add_tail(&task->t_state_list, &dev->state_task_list);
1030
1031         atomic_set(&task->task_state_active, 1);
1032
1033         DEBUG_TSTATE("Added ITT: 0x%08x task[%p] to dev: %p\n",
1034                 task->task_se_cmd->se_tfo->get_task_tag(task->task_se_cmd),
1035                 task, dev);
1036 }
1037
1038 static void transport_add_tasks_to_state_queue(struct se_cmd *cmd)
1039 {
1040         struct se_device *dev;
1041         struct se_task *task;
1042         unsigned long flags;
1043
1044         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
1045         list_for_each_entry(task, &cmd->t_task.t_task_list, t_list) {
1046                 dev = task->se_dev;
1047
1048                 if (atomic_read(&task->task_state_active))
1049                         continue;
1050
1051                 spin_lock(&dev->execute_task_lock);
1052                 list_add_tail(&task->t_state_list, &dev->state_task_list);
1053                 atomic_set(&task->task_state_active, 1);
1054
1055                 DEBUG_TSTATE("Added ITT: 0x%08x task[%p] to dev: %p\n",
1056                         task->se_cmd->se_tfo->get_task_tag(
1057                         task->task_se_cmd), task, dev);
1058
1059                 spin_unlock(&dev->execute_task_lock);
1060         }
1061         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
1062 }
1063
1064 static void transport_add_tasks_from_cmd(struct se_cmd *cmd)
1065 {
1066         struct se_device *dev = cmd->se_dev;
1067         struct se_task *task, *task_prev = NULL;
1068         unsigned long flags;
1069
1070         spin_lock_irqsave(&dev->execute_task_lock, flags);
1071         list_for_each_entry(task, &cmd->t_task.t_task_list, t_list) {
1072                 if (atomic_read(&task->task_execute_queue))
1073                         continue;
1074                 /*
1075                  * __transport_add_task_to_execute_queue() handles the
1076                  * SAM Task Attribute emulation if enabled
1077                  */
1078                 __transport_add_task_to_execute_queue(task, task_prev, dev);
1079                 atomic_set(&task->task_execute_queue, 1);
1080                 task_prev = task;
1081         }
1082         spin_unlock_irqrestore(&dev->execute_task_lock, flags);
1083 }
1084
1085 /*      transport_remove_task_from_execute_queue():
1086  *
1087  *
1088  */
1089 void transport_remove_task_from_execute_queue(
1090         struct se_task *task,
1091         struct se_device *dev)
1092 {
1093         unsigned long flags;
1094
1095         if (atomic_read(&task->task_execute_queue) == 0) {
1096                 dump_stack();
1097                 return;
1098         }
1099
1100         spin_lock_irqsave(&dev->execute_task_lock, flags);
1101         list_del(&task->t_execute_list);
1102         atomic_set(&task->task_execute_queue, 0);
1103         atomic_dec(&dev->execute_tasks);
1104         spin_unlock_irqrestore(&dev->execute_task_lock, flags);
1105 }
1106
1107 unsigned char *transport_dump_cmd_direction(struct se_cmd *cmd)
1108 {
1109         switch (cmd->data_direction) {
1110         case DMA_NONE:
1111                 return "NONE";
1112         case DMA_FROM_DEVICE:
1113                 return "READ";
1114         case DMA_TO_DEVICE:
1115                 return "WRITE";
1116         case DMA_BIDIRECTIONAL:
1117                 return "BIDI";
1118         default:
1119                 break;
1120         }
1121
1122         return "UNKNOWN";
1123 }
1124
1125 void transport_dump_dev_state(
1126         struct se_device *dev,
1127         char *b,
1128         int *bl)
1129 {
1130         *bl += sprintf(b + *bl, "Status: ");
1131         switch (dev->dev_status) {
1132         case TRANSPORT_DEVICE_ACTIVATED:
1133                 *bl += sprintf(b + *bl, "ACTIVATED");
1134                 break;
1135         case TRANSPORT_DEVICE_DEACTIVATED:
1136                 *bl += sprintf(b + *bl, "DEACTIVATED");
1137                 break;
1138         case TRANSPORT_DEVICE_SHUTDOWN:
1139                 *bl += sprintf(b + *bl, "SHUTDOWN");
1140                 break;
1141         case TRANSPORT_DEVICE_OFFLINE_ACTIVATED:
1142         case TRANSPORT_DEVICE_OFFLINE_DEACTIVATED:
1143                 *bl += sprintf(b + *bl, "OFFLINE");
1144                 break;
1145         default:
1146                 *bl += sprintf(b + *bl, "UNKNOWN=%d", dev->dev_status);
1147                 break;
1148         }
1149
1150         *bl += sprintf(b + *bl, "  Execute/Left/Max Queue Depth: %d/%d/%d",
1151                 atomic_read(&dev->execute_tasks), atomic_read(&dev->depth_left),
1152                 dev->queue_depth);
1153         *bl += sprintf(b + *bl, "  SectorSize: %u  MaxSectors: %u\n",
1154                 dev->se_sub_dev->se_dev_attrib.block_size, dev->se_sub_dev->se_dev_attrib.max_sectors);
1155         *bl += sprintf(b + *bl, "        ");
1156 }
1157
1158 /*      transport_release_all_cmds():
1159  *
1160  *
1161  */
1162 static void transport_release_all_cmds(struct se_device *dev)
1163 {
1164         struct se_cmd *cmd, *tcmd;
1165         int bug_out = 0, t_state;
1166         unsigned long flags;
1167
1168         spin_lock_irqsave(&dev->dev_queue_obj.cmd_queue_lock, flags);
1169         list_for_each_entry_safe(cmd, tcmd, &dev->dev_queue_obj.qobj_list,
1170                                 se_queue_node) {
1171                 t_state = cmd->t_state;
1172                 list_del(&cmd->se_queue_node);
1173                 spin_unlock_irqrestore(&dev->dev_queue_obj.cmd_queue_lock,
1174                                 flags);
1175
1176                 printk(KERN_ERR "Releasing ITT: 0x%08x, i_state: %u,"
1177                         " t_state: %u directly\n",
1178                         cmd->se_tfo->get_task_tag(cmd),
1179                         cmd->se_tfo->get_cmd_state(cmd), t_state);
1180
1181                 transport_release_fe_cmd(cmd);
1182                 bug_out = 1;
1183
1184                 spin_lock_irqsave(&dev->dev_queue_obj.cmd_queue_lock, flags);
1185         }
1186         spin_unlock_irqrestore(&dev->dev_queue_obj.cmd_queue_lock, flags);
1187 #if 0
1188         if (bug_out)
1189                 BUG();
1190 #endif
1191 }
1192
1193 void transport_dump_vpd_proto_id(
1194         struct t10_vpd *vpd,
1195         unsigned char *p_buf,
1196         int p_buf_len)
1197 {
1198         unsigned char buf[VPD_TMP_BUF_SIZE];
1199         int len;
1200
1201         memset(buf, 0, VPD_TMP_BUF_SIZE);
1202         len = sprintf(buf, "T10 VPD Protocol Identifier: ");
1203
1204         switch (vpd->protocol_identifier) {
1205         case 0x00:
1206                 sprintf(buf+len, "Fibre Channel\n");
1207                 break;
1208         case 0x10:
1209                 sprintf(buf+len, "Parallel SCSI\n");
1210                 break;
1211         case 0x20:
1212                 sprintf(buf+len, "SSA\n");
1213                 break;
1214         case 0x30:
1215                 sprintf(buf+len, "IEEE 1394\n");
1216                 break;
1217         case 0x40:
1218                 sprintf(buf+len, "SCSI Remote Direct Memory Access"
1219                                 " Protocol\n");
1220                 break;
1221         case 0x50:
1222                 sprintf(buf+len, "Internet SCSI (iSCSI)\n");
1223                 break;
1224         case 0x60:
1225                 sprintf(buf+len, "SAS Serial SCSI Protocol\n");
1226                 break;
1227         case 0x70:
1228                 sprintf(buf+len, "Automation/Drive Interface Transport"
1229                                 " Protocol\n");
1230                 break;
1231         case 0x80:
1232                 sprintf(buf+len, "AT Attachment Interface ATA/ATAPI\n");
1233                 break;
1234         default:
1235                 sprintf(buf+len, "Unknown 0x%02x\n",
1236                                 vpd->protocol_identifier);
1237                 break;
1238         }
1239
1240         if (p_buf)
1241                 strncpy(p_buf, buf, p_buf_len);
1242         else
1243                 printk(KERN_INFO "%s", buf);
1244 }
1245
1246 void
1247 transport_set_vpd_proto_id(struct t10_vpd *vpd, unsigned char *page_83)
1248 {
1249         /*
1250          * Check if the Protocol Identifier Valid (PIV) bit is set..
1251          *
1252          * from spc3r23.pdf section 7.5.1
1253          */
1254          if (page_83[1] & 0x80) {
1255                 vpd->protocol_identifier = (page_83[0] & 0xf0);
1256                 vpd->protocol_identifier_set = 1;
1257                 transport_dump_vpd_proto_id(vpd, NULL, 0);
1258         }
1259 }
1260 EXPORT_SYMBOL(transport_set_vpd_proto_id);
1261
1262 int transport_dump_vpd_assoc(
1263         struct t10_vpd *vpd,
1264         unsigned char *p_buf,
1265         int p_buf_len)
1266 {
1267         unsigned char buf[VPD_TMP_BUF_SIZE];
1268         int ret = 0;
1269         int len;
1270
1271         memset(buf, 0, VPD_TMP_BUF_SIZE);
1272         len = sprintf(buf, "T10 VPD Identifier Association: ");
1273
1274         switch (vpd->association) {
1275         case 0x00:
1276                 sprintf(buf+len, "addressed logical unit\n");
1277                 break;
1278         case 0x10:
1279                 sprintf(buf+len, "target port\n");
1280                 break;
1281         case 0x20:
1282                 sprintf(buf+len, "SCSI target device\n");
1283                 break;
1284         default:
1285                 sprintf(buf+len, "Unknown 0x%02x\n", vpd->association);
1286                 ret = -EINVAL;
1287                 break;
1288         }
1289
1290         if (p_buf)
1291                 strncpy(p_buf, buf, p_buf_len);
1292         else
1293                 printk("%s", buf);
1294
1295         return ret;
1296 }
1297
1298 int transport_set_vpd_assoc(struct t10_vpd *vpd, unsigned char *page_83)
1299 {
1300         /*
1301          * The VPD identification association..
1302          *
1303          * from spc3r23.pdf Section 7.6.3.1 Table 297
1304          */
1305         vpd->association = (page_83[1] & 0x30);
1306         return transport_dump_vpd_assoc(vpd, NULL, 0);
1307 }
1308 EXPORT_SYMBOL(transport_set_vpd_assoc);
1309
1310 int transport_dump_vpd_ident_type(
1311         struct t10_vpd *vpd,
1312         unsigned char *p_buf,
1313         int p_buf_len)
1314 {
1315         unsigned char buf[VPD_TMP_BUF_SIZE];
1316         int ret = 0;
1317         int len;
1318
1319         memset(buf, 0, VPD_TMP_BUF_SIZE);
1320         len = sprintf(buf, "T10 VPD Identifier Type: ");
1321
1322         switch (vpd->device_identifier_type) {
1323         case 0x00:
1324                 sprintf(buf+len, "Vendor specific\n");
1325                 break;
1326         case 0x01:
1327                 sprintf(buf+len, "T10 Vendor ID based\n");
1328                 break;
1329         case 0x02:
1330                 sprintf(buf+len, "EUI-64 based\n");
1331                 break;
1332         case 0x03:
1333                 sprintf(buf+len, "NAA\n");
1334                 break;
1335         case 0x04:
1336                 sprintf(buf+len, "Relative target port identifier\n");
1337                 break;
1338         case 0x08:
1339                 sprintf(buf+len, "SCSI name string\n");
1340                 break;
1341         default:
1342                 sprintf(buf+len, "Unsupported: 0x%02x\n",
1343                                 vpd->device_identifier_type);
1344                 ret = -EINVAL;
1345                 break;
1346         }
1347
1348         if (p_buf) {
1349                 if (p_buf_len < strlen(buf)+1)
1350                         return -EINVAL;
1351                 strncpy(p_buf, buf, p_buf_len);
1352         } else {
1353                 printk("%s", buf);
1354         }
1355
1356         return ret;
1357 }
1358
1359 int transport_set_vpd_ident_type(struct t10_vpd *vpd, unsigned char *page_83)
1360 {
1361         /*
1362          * The VPD identifier type..
1363          *
1364          * from spc3r23.pdf Section 7.6.3.1 Table 298
1365          */
1366         vpd->device_identifier_type = (page_83[1] & 0x0f);
1367         return transport_dump_vpd_ident_type(vpd, NULL, 0);
1368 }
1369 EXPORT_SYMBOL(transport_set_vpd_ident_type);
1370
1371 int transport_dump_vpd_ident(
1372         struct t10_vpd *vpd,
1373         unsigned char *p_buf,
1374         int p_buf_len)
1375 {
1376         unsigned char buf[VPD_TMP_BUF_SIZE];
1377         int ret = 0;
1378
1379         memset(buf, 0, VPD_TMP_BUF_SIZE);
1380
1381         switch (vpd->device_identifier_code_set) {
1382         case 0x01: /* Binary */
1383                 sprintf(buf, "T10 VPD Binary Device Identifier: %s\n",
1384                         &vpd->device_identifier[0]);
1385                 break;
1386         case 0x02: /* ASCII */
1387                 sprintf(buf, "T10 VPD ASCII Device Identifier: %s\n",
1388                         &vpd->device_identifier[0]);
1389                 break;
1390         case 0x03: /* UTF-8 */
1391                 sprintf(buf, "T10 VPD UTF-8 Device Identifier: %s\n",
1392                         &vpd->device_identifier[0]);
1393                 break;
1394         default:
1395                 sprintf(buf, "T10 VPD Device Identifier encoding unsupported:"
1396                         " 0x%02x", vpd->device_identifier_code_set);
1397                 ret = -EINVAL;
1398                 break;
1399         }
1400
1401         if (p_buf)
1402                 strncpy(p_buf, buf, p_buf_len);
1403         else
1404                 printk("%s", buf);
1405
1406         return ret;
1407 }
1408
1409 int
1410 transport_set_vpd_ident(struct t10_vpd *vpd, unsigned char *page_83)
1411 {
1412         static const char hex_str[] = "0123456789abcdef";
1413         int j = 0, i = 4; /* offset to start of the identifer */
1414
1415         /*
1416          * The VPD Code Set (encoding)
1417          *
1418          * from spc3r23.pdf Section 7.6.3.1 Table 296
1419          */
1420         vpd->device_identifier_code_set = (page_83[0] & 0x0f);
1421         switch (vpd->device_identifier_code_set) {
1422         case 0x01: /* Binary */
1423                 vpd->device_identifier[j++] =
1424                                 hex_str[vpd->device_identifier_type];
1425                 while (i < (4 + page_83[3])) {
1426                         vpd->device_identifier[j++] =
1427                                 hex_str[(page_83[i] & 0xf0) >> 4];
1428                         vpd->device_identifier[j++] =
1429                                 hex_str[page_83[i] & 0x0f];
1430                         i++;
1431                 }
1432                 break;
1433         case 0x02: /* ASCII */
1434         case 0x03: /* UTF-8 */
1435                 while (i < (4 + page_83[3]))
1436                         vpd->device_identifier[j++] = page_83[i++];
1437                 break;
1438         default:
1439                 break;
1440         }
1441
1442         return transport_dump_vpd_ident(vpd, NULL, 0);
1443 }
1444 EXPORT_SYMBOL(transport_set_vpd_ident);
1445
1446 static void core_setup_task_attr_emulation(struct se_device *dev)
1447 {
1448         /*
1449          * If this device is from Target_Core_Mod/pSCSI, disable the
1450          * SAM Task Attribute emulation.
1451          *
1452          * This is currently not available in upsream Linux/SCSI Target
1453          * mode code, and is assumed to be disabled while using TCM/pSCSI.
1454          */
1455         if (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV) {
1456                 dev->dev_task_attr_type = SAM_TASK_ATTR_PASSTHROUGH;
1457                 return;
1458         }
1459
1460         dev->dev_task_attr_type = SAM_TASK_ATTR_EMULATED;
1461         DEBUG_STA("%s: Using SAM_TASK_ATTR_EMULATED for SPC: 0x%02x"
1462                 " device\n", dev->transport->name,
1463                 dev->transport->get_device_rev(dev));
1464 }
1465
1466 static void scsi_dump_inquiry(struct se_device *dev)
1467 {
1468         struct t10_wwn *wwn = &dev->se_sub_dev->t10_wwn;
1469         int i, device_type;
1470         /*
1471          * Print Linux/SCSI style INQUIRY formatting to the kernel ring buffer
1472          */
1473         printk("  Vendor: ");
1474         for (i = 0; i < 8; i++)
1475                 if (wwn->vendor[i] >= 0x20)
1476                         printk("%c", wwn->vendor[i]);
1477                 else
1478                         printk(" ");
1479
1480         printk("  Model: ");
1481         for (i = 0; i < 16; i++)
1482                 if (wwn->model[i] >= 0x20)
1483                         printk("%c", wwn->model[i]);
1484                 else
1485                         printk(" ");
1486
1487         printk("  Revision: ");
1488         for (i = 0; i < 4; i++)
1489                 if (wwn->revision[i] >= 0x20)
1490                         printk("%c", wwn->revision[i]);
1491                 else
1492                         printk(" ");
1493
1494         printk("\n");
1495
1496         device_type = dev->transport->get_device_type(dev);
1497         printk("  Type:   %s ", scsi_device_type(device_type));
1498         printk("                 ANSI SCSI revision: %02x\n",
1499                                 dev->transport->get_device_rev(dev));
1500 }
1501
1502 struct se_device *transport_add_device_to_core_hba(
1503         struct se_hba *hba,
1504         struct se_subsystem_api *transport,
1505         struct se_subsystem_dev *se_dev,
1506         u32 device_flags,
1507         void *transport_dev,
1508         struct se_dev_limits *dev_limits,
1509         const char *inquiry_prod,
1510         const char *inquiry_rev)
1511 {
1512         int force_pt;
1513         struct se_device  *dev;
1514
1515         dev = kzalloc(sizeof(struct se_device), GFP_KERNEL);
1516         if (!(dev)) {
1517                 printk(KERN_ERR "Unable to allocate memory for se_dev_t\n");
1518                 return NULL;
1519         }
1520
1521         transport_init_queue_obj(&dev->dev_queue_obj);
1522         dev->dev_flags          = device_flags;
1523         dev->dev_status         |= TRANSPORT_DEVICE_DEACTIVATED;
1524         dev->dev_ptr            = transport_dev;
1525         dev->se_hba             = hba;
1526         dev->se_sub_dev         = se_dev;
1527         dev->transport          = transport;
1528         atomic_set(&dev->active_cmds, 0);
1529         INIT_LIST_HEAD(&dev->dev_list);
1530         INIT_LIST_HEAD(&dev->dev_sep_list);
1531         INIT_LIST_HEAD(&dev->dev_tmr_list);
1532         INIT_LIST_HEAD(&dev->execute_task_list);
1533         INIT_LIST_HEAD(&dev->delayed_cmd_list);
1534         INIT_LIST_HEAD(&dev->ordered_cmd_list);
1535         INIT_LIST_HEAD(&dev->state_task_list);
1536         spin_lock_init(&dev->execute_task_lock);
1537         spin_lock_init(&dev->delayed_cmd_lock);
1538         spin_lock_init(&dev->ordered_cmd_lock);
1539         spin_lock_init(&dev->state_task_lock);
1540         spin_lock_init(&dev->dev_alua_lock);
1541         spin_lock_init(&dev->dev_reservation_lock);
1542         spin_lock_init(&dev->dev_status_lock);
1543         spin_lock_init(&dev->dev_status_thr_lock);
1544         spin_lock_init(&dev->se_port_lock);
1545         spin_lock_init(&dev->se_tmr_lock);
1546
1547         dev->queue_depth        = dev_limits->queue_depth;
1548         atomic_set(&dev->depth_left, dev->queue_depth);
1549         atomic_set(&dev->dev_ordered_id, 0);
1550
1551         se_dev_set_default_attribs(dev, dev_limits);
1552
1553         dev->dev_index = scsi_get_new_index(SCSI_DEVICE_INDEX);
1554         dev->creation_time = get_jiffies_64();
1555         spin_lock_init(&dev->stats_lock);
1556
1557         spin_lock(&hba->device_lock);
1558         list_add_tail(&dev->dev_list, &hba->hba_dev_list);
1559         hba->dev_count++;
1560         spin_unlock(&hba->device_lock);
1561         /*
1562          * Setup the SAM Task Attribute emulation for struct se_device
1563          */
1564         core_setup_task_attr_emulation(dev);
1565         /*
1566          * Force PR and ALUA passthrough emulation with internal object use.
1567          */
1568         force_pt = (hba->hba_flags & HBA_FLAGS_INTERNAL_USE);
1569         /*
1570          * Setup the Reservations infrastructure for struct se_device
1571          */
1572         core_setup_reservations(dev, force_pt);
1573         /*
1574          * Setup the Asymmetric Logical Unit Assignment for struct se_device
1575          */
1576         if (core_setup_alua(dev, force_pt) < 0)
1577                 goto out;
1578
1579         /*
1580          * Startup the struct se_device processing thread
1581          */
1582         dev->process_thread = kthread_run(transport_processing_thread, dev,
1583                                           "LIO_%s", dev->transport->name);
1584         if (IS_ERR(dev->process_thread)) {
1585                 printk(KERN_ERR "Unable to create kthread: LIO_%s\n",
1586                         dev->transport->name);
1587                 goto out;
1588         }
1589
1590         /*
1591          * Preload the initial INQUIRY const values if we are doing
1592          * anything virtual (IBLOCK, FILEIO, RAMDISK), but not for TCM/pSCSI
1593          * passthrough because this is being provided by the backend LLD.
1594          * This is required so that transport_get_inquiry() copies these
1595          * originals once back into DEV_T10_WWN(dev) for the virtual device
1596          * setup.
1597          */
1598         if (dev->transport->transport_type != TRANSPORT_PLUGIN_PHBA_PDEV) {
1599                 if (!inquiry_prod || !inquiry_rev) {
1600                         printk(KERN_ERR "All non TCM/pSCSI plugins require"
1601                                 " INQUIRY consts\n");
1602                         goto out;
1603                 }
1604
1605                 strncpy(&dev->se_sub_dev->t10_wwn.vendor[0], "LIO-ORG", 8);
1606                 strncpy(&dev->se_sub_dev->t10_wwn.model[0], inquiry_prod, 16);
1607                 strncpy(&dev->se_sub_dev->t10_wwn.revision[0], inquiry_rev, 4);
1608         }
1609         scsi_dump_inquiry(dev);
1610
1611         return dev;
1612 out:
1613         kthread_stop(dev->process_thread);
1614
1615         spin_lock(&hba->device_lock);
1616         list_del(&dev->dev_list);
1617         hba->dev_count--;
1618         spin_unlock(&hba->device_lock);
1619
1620         se_release_vpd_for_dev(dev);
1621
1622         kfree(dev);
1623
1624         return NULL;
1625 }
1626 EXPORT_SYMBOL(transport_add_device_to_core_hba);
1627
1628 /*      transport_generic_prepare_cdb():
1629  *
1630  *      Since the Initiator sees iSCSI devices as LUNs,  the SCSI CDB will
1631  *      contain the iSCSI LUN in bits 7-5 of byte 1 as per SAM-2.
1632  *      The point of this is since we are mapping iSCSI LUNs to
1633  *      SCSI Target IDs having a non-zero LUN in the CDB will throw the
1634  *      devices and HBAs for a loop.
1635  */
1636 static inline void transport_generic_prepare_cdb(
1637         unsigned char *cdb)
1638 {
1639         switch (cdb[0]) {
1640         case READ_10: /* SBC - RDProtect */
1641         case READ_12: /* SBC - RDProtect */
1642         case READ_16: /* SBC - RDProtect */
1643         case SEND_DIAGNOSTIC: /* SPC - SELF-TEST Code */
1644         case VERIFY: /* SBC - VRProtect */
1645         case VERIFY_16: /* SBC - VRProtect */
1646         case WRITE_VERIFY: /* SBC - VRProtect */
1647         case WRITE_VERIFY_12: /* SBC - VRProtect */
1648                 break;
1649         default:
1650                 cdb[1] &= 0x1f; /* clear logical unit number */
1651                 break;
1652         }
1653 }
1654
1655 static struct se_task *
1656 transport_generic_get_task(struct se_cmd *cmd,
1657                 enum dma_data_direction data_direction)
1658 {
1659         struct se_task *task;
1660         struct se_device *dev = cmd->se_dev;
1661         unsigned long flags;
1662
1663         task = dev->transport->alloc_task(cmd);
1664         if (!task) {
1665                 printk(KERN_ERR "Unable to allocate struct se_task\n");
1666                 return NULL;
1667         }
1668
1669         INIT_LIST_HEAD(&task->t_list);
1670         INIT_LIST_HEAD(&task->t_execute_list);
1671         INIT_LIST_HEAD(&task->t_state_list);
1672         init_completion(&task->task_stop_comp);
1673         task->task_no = cmd->t_task.t_tasks_no++;
1674         task->task_se_cmd = cmd;
1675         task->se_dev = dev;
1676         task->task_data_direction = data_direction;
1677
1678         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
1679         list_add_tail(&task->t_list, &cmd->t_task.t_task_list);
1680         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
1681
1682         return task;
1683 }
1684
1685 static int transport_generic_cmd_sequencer(struct se_cmd *, unsigned char *);
1686
1687 /*
1688  * Used by fabric modules containing a local struct se_cmd within their
1689  * fabric dependent per I/O descriptor.
1690  */
1691 void transport_init_se_cmd(
1692         struct se_cmd *cmd,
1693         struct target_core_fabric_ops *tfo,
1694         struct se_session *se_sess,
1695         u32 data_length,
1696         int data_direction,
1697         int task_attr,
1698         unsigned char *sense_buffer)
1699 {
1700         INIT_LIST_HEAD(&cmd->se_lun_node);
1701         INIT_LIST_HEAD(&cmd->se_delayed_node);
1702         INIT_LIST_HEAD(&cmd->se_ordered_node);
1703
1704         INIT_LIST_HEAD(&cmd->t_task.t_mem_list);
1705         INIT_LIST_HEAD(&cmd->t_task.t_mem_bidi_list);
1706         INIT_LIST_HEAD(&cmd->t_task.t_task_list);
1707         init_completion(&cmd->t_task.transport_lun_fe_stop_comp);
1708         init_completion(&cmd->t_task.transport_lun_stop_comp);
1709         init_completion(&cmd->t_task.t_transport_stop_comp);
1710         spin_lock_init(&cmd->t_task.t_state_lock);
1711         atomic_set(&cmd->t_task.transport_dev_active, 1);
1712
1713         cmd->se_tfo = tfo;
1714         cmd->se_sess = se_sess;
1715         cmd->data_length = data_length;
1716         cmd->data_direction = data_direction;
1717         cmd->sam_task_attr = task_attr;
1718         cmd->sense_buffer = sense_buffer;
1719 }
1720 EXPORT_SYMBOL(transport_init_se_cmd);
1721
1722 static int transport_check_alloc_task_attr(struct se_cmd *cmd)
1723 {
1724         /*
1725          * Check if SAM Task Attribute emulation is enabled for this
1726          * struct se_device storage object
1727          */
1728         if (cmd->se_dev->dev_task_attr_type != SAM_TASK_ATTR_EMULATED)
1729                 return 0;
1730
1731         if (cmd->sam_task_attr == MSG_ACA_TAG) {
1732                 DEBUG_STA("SAM Task Attribute ACA"
1733                         " emulation is not supported\n");
1734                 return -EINVAL;
1735         }
1736         /*
1737          * Used to determine when ORDERED commands should go from
1738          * Dormant to Active status.
1739          */
1740         cmd->se_ordered_id = atomic_inc_return(&cmd->se_dev->dev_ordered_id);
1741         smp_mb__after_atomic_inc();
1742         DEBUG_STA("Allocated se_ordered_id: %u for Task Attr: 0x%02x on %s\n",
1743                         cmd->se_ordered_id, cmd->sam_task_attr,
1744                         TRANSPORT(cmd->se_dev)->name);
1745         return 0;
1746 }
1747
1748 void transport_free_se_cmd(
1749         struct se_cmd *se_cmd)
1750 {
1751         if (se_cmd->se_tmr_req)
1752                 core_tmr_release_req(se_cmd->se_tmr_req);
1753         /*
1754          * Check and free any extended CDB buffer that was allocated
1755          */
1756         if (se_cmd->t_task.t_task_cdb != se_cmd->t_task.__t_task_cdb)
1757                 kfree(se_cmd->t_task.t_task_cdb);
1758 }
1759 EXPORT_SYMBOL(transport_free_se_cmd);
1760
1761 static void transport_generic_wait_for_tasks(struct se_cmd *, int, int);
1762
1763 /*      transport_generic_allocate_tasks():
1764  *
1765  *      Called from fabric RX Thread.
1766  */
1767 int transport_generic_allocate_tasks(
1768         struct se_cmd *cmd,
1769         unsigned char *cdb)
1770 {
1771         int ret;
1772
1773         transport_generic_prepare_cdb(cdb);
1774
1775         /*
1776          * This is needed for early exceptions.
1777          */
1778         cmd->transport_wait_for_tasks = &transport_generic_wait_for_tasks;
1779
1780         /*
1781          * Ensure that the received CDB is less than the max (252 + 8) bytes
1782          * for VARIABLE_LENGTH_CMD
1783          */
1784         if (scsi_command_size(cdb) > SCSI_MAX_VARLEN_CDB_SIZE) {
1785                 printk(KERN_ERR "Received SCSI CDB with command_size: %d that"
1786                         " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n",
1787                         scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE);
1788                 return -EINVAL;
1789         }
1790         /*
1791          * If the received CDB is larger than TCM_MAX_COMMAND_SIZE,
1792          * allocate the additional extended CDB buffer now..  Otherwise
1793          * setup the pointer from __t_task_cdb to t_task_cdb.
1794          */
1795         if (scsi_command_size(cdb) > sizeof(cmd->t_task.__t_task_cdb)) {
1796                 cmd->t_task.t_task_cdb = kzalloc(scsi_command_size(cdb),
1797                                                 GFP_KERNEL);
1798                 if (!(cmd->t_task.t_task_cdb)) {
1799                         printk(KERN_ERR "Unable to allocate cmd->t_task.t_task_cdb"
1800                                 " %u > sizeof(cmd->t_task.__t_task_cdb): %lu ops\n",
1801                                 scsi_command_size(cdb),
1802                                 (unsigned long)sizeof(cmd->t_task.__t_task_cdb));
1803                         return -ENOMEM;
1804                 }
1805         } else
1806                 cmd->t_task.t_task_cdb = &cmd->t_task.__t_task_cdb[0];
1807         /*
1808          * Copy the original CDB into cmd->t_task.
1809          */
1810         memcpy(cmd->t_task.t_task_cdb, cdb, scsi_command_size(cdb));
1811         /*
1812          * Setup the received CDB based on SCSI defined opcodes and
1813          * perform unit attention, persistent reservations and ALUA
1814          * checks for virtual device backends.  The cmd->t_task.t_task_cdb
1815          * pointer is expected to be setup before we reach this point.
1816          */
1817         ret = transport_generic_cmd_sequencer(cmd, cdb);
1818         if (ret < 0)
1819                 return ret;
1820         /*
1821          * Check for SAM Task Attribute Emulation
1822          */
1823         if (transport_check_alloc_task_attr(cmd) < 0) {
1824                 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
1825                 cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD;
1826                 return -EINVAL;
1827         }
1828         spin_lock(&cmd->se_lun->lun_sep_lock);
1829         if (cmd->se_lun->lun_sep)
1830                 cmd->se_lun->lun_sep->sep_stats.cmd_pdus++;
1831         spin_unlock(&cmd->se_lun->lun_sep_lock);
1832         return 0;
1833 }
1834 EXPORT_SYMBOL(transport_generic_allocate_tasks);
1835
1836 /*
1837  * Used by fabric module frontends not defining a TFO->new_cmd_map()
1838  * to queue up a newly setup se_cmd w/ TRANSPORT_NEW_CMD statis
1839  */
1840 int transport_generic_handle_cdb(
1841         struct se_cmd *cmd)
1842 {
1843         if (!cmd->se_lun) {
1844                 dump_stack();
1845                 printk(KERN_ERR "cmd->se_lun is NULL\n");
1846                 return -EINVAL;
1847         }
1848
1849         transport_add_cmd_to_queue(cmd, TRANSPORT_NEW_CMD);
1850         return 0;
1851 }
1852 EXPORT_SYMBOL(transport_generic_handle_cdb);
1853
1854 /*
1855  * Used by fabric module frontends defining a TFO->new_cmd_map() caller
1856  * to  queue up a newly setup se_cmd w/ TRANSPORT_NEW_CMD_MAP in order to
1857  * complete setup in TCM process context w/ TFO->new_cmd_map().
1858  */
1859 int transport_generic_handle_cdb_map(
1860         struct se_cmd *cmd)
1861 {
1862         if (!cmd->se_lun) {
1863                 dump_stack();
1864                 printk(KERN_ERR "cmd->se_lun is NULL\n");
1865                 return -EINVAL;
1866         }
1867
1868         transport_add_cmd_to_queue(cmd, TRANSPORT_NEW_CMD_MAP);
1869         return 0;
1870 }
1871 EXPORT_SYMBOL(transport_generic_handle_cdb_map);
1872
1873 /*      transport_generic_handle_data():
1874  *
1875  *
1876  */
1877 int transport_generic_handle_data(
1878         struct se_cmd *cmd)
1879 {
1880         /*
1881          * For the software fabric case, then we assume the nexus is being
1882          * failed/shutdown when signals are pending from the kthread context
1883          * caller, so we return a failure.  For the HW target mode case running
1884          * in interrupt code, the signal_pending() check is skipped.
1885          */
1886         if (!in_interrupt() && signal_pending(current))
1887                 return -EPERM;
1888         /*
1889          * If the received CDB has aleady been ABORTED by the generic
1890          * target engine, we now call transport_check_aborted_status()
1891          * to queue any delated TASK_ABORTED status for the received CDB to the
1892          * fabric module as we are expecting no further incoming DATA OUT
1893          * sequences at this point.
1894          */
1895         if (transport_check_aborted_status(cmd, 1) != 0)
1896                 return 0;
1897
1898         transport_add_cmd_to_queue(cmd, TRANSPORT_PROCESS_WRITE);
1899         return 0;
1900 }
1901 EXPORT_SYMBOL(transport_generic_handle_data);
1902
1903 /*      transport_generic_handle_tmr():
1904  *
1905  *
1906  */
1907 int transport_generic_handle_tmr(
1908         struct se_cmd *cmd)
1909 {
1910         /*
1911          * This is needed for early exceptions.
1912          */
1913         cmd->transport_wait_for_tasks = &transport_generic_wait_for_tasks;
1914
1915         transport_add_cmd_to_queue(cmd, TRANSPORT_PROCESS_TMR);
1916         return 0;
1917 }
1918 EXPORT_SYMBOL(transport_generic_handle_tmr);
1919
1920 void transport_generic_free_cmd_intr(
1921         struct se_cmd *cmd)
1922 {
1923         transport_add_cmd_to_queue(cmd, TRANSPORT_FREE_CMD_INTR);
1924 }
1925 EXPORT_SYMBOL(transport_generic_free_cmd_intr);
1926
1927 static int transport_stop_tasks_for_cmd(struct se_cmd *cmd)
1928 {
1929         struct se_task *task, *task_tmp;
1930         unsigned long flags;
1931         int ret = 0;
1932
1933         DEBUG_TS("ITT[0x%08x] - Stopping tasks\n",
1934                 cmd->se_tfo->get_task_tag(cmd));
1935
1936         /*
1937          * No tasks remain in the execution queue
1938          */
1939         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
1940         list_for_each_entry_safe(task, task_tmp,
1941                                 &cmd->t_task.t_task_list, t_list) {
1942                 DEBUG_TS("task_no[%d] - Processing task %p\n",
1943                                 task->task_no, task);
1944                 /*
1945                  * If the struct se_task has not been sent and is not active,
1946                  * remove the struct se_task from the execution queue.
1947                  */
1948                 if (!atomic_read(&task->task_sent) &&
1949                     !atomic_read(&task->task_active)) {
1950                         spin_unlock_irqrestore(&cmd->t_task.t_state_lock,
1951                                         flags);
1952                         transport_remove_task_from_execute_queue(task,
1953                                         task->se_dev);
1954
1955                         DEBUG_TS("task_no[%d] - Removed from execute queue\n",
1956                                 task->task_no);
1957                         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
1958                         continue;
1959                 }
1960
1961                 /*
1962                  * If the struct se_task is active, sleep until it is returned
1963                  * from the plugin.
1964                  */
1965                 if (atomic_read(&task->task_active)) {
1966                         atomic_set(&task->task_stop, 1);
1967                         spin_unlock_irqrestore(&cmd->t_task.t_state_lock,
1968                                         flags);
1969
1970                         DEBUG_TS("task_no[%d] - Waiting to complete\n",
1971                                 task->task_no);
1972                         wait_for_completion(&task->task_stop_comp);
1973                         DEBUG_TS("task_no[%d] - Stopped successfully\n",
1974                                 task->task_no);
1975
1976                         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
1977                         atomic_dec(&cmd->t_task.t_task_cdbs_left);
1978
1979                         atomic_set(&task->task_active, 0);
1980                         atomic_set(&task->task_stop, 0);
1981                 } else {
1982                         DEBUG_TS("task_no[%d] - Did nothing\n", task->task_no);
1983                         ret++;
1984                 }
1985
1986                 __transport_stop_task_timer(task, &flags);
1987         }
1988         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
1989
1990         return ret;
1991 }
1992
1993 /*
1994  * Handle SAM-esque emulation for generic transport request failures.
1995  */
1996 static void transport_generic_request_failure(
1997         struct se_cmd *cmd,
1998         struct se_device *dev,
1999         int complete,
2000         int sc)
2001 {
2002         DEBUG_GRF("-----[ Storage Engine Exception for cmd: %p ITT: 0x%08x"
2003                 " CDB: 0x%02x\n", cmd, cmd->se_tfo->get_task_tag(cmd),
2004                 cmd->t_task.t_task_cdb[0]);
2005         DEBUG_GRF("-----[ i_state: %d t_state/def_t_state:"
2006                 " %d/%d transport_error_status: %d\n",
2007                 cmd->se_tfo->get_cmd_state(cmd),
2008                 cmd->t_state, cmd->deferred_t_state,
2009                 cmd->transport_error_status);
2010         DEBUG_GRF("-----[ t_task_cdbs: %d t_task_cdbs_left: %d"
2011                 " t_task_cdbs_sent: %d t_task_cdbs_ex_left: %d --"
2012                 " t_transport_active: %d t_transport_stop: %d"
2013                 " t_transport_sent: %d\n", cmd->t_task.t_task_cdbs,
2014                 atomic_read(&cmd->t_task.t_task_cdbs_left),
2015                 atomic_read(&cmd->t_task.t_task_cdbs_sent),
2016                 atomic_read(&cmd->t_task.t_task_cdbs_ex_left),
2017                 atomic_read(&cmd->t_task.t_transport_active),
2018                 atomic_read(&cmd->t_task.t_transport_stop),
2019                 atomic_read(&cmd->t_task.t_transport_sent));
2020
2021         transport_stop_all_task_timers(cmd);
2022
2023         if (dev)
2024                 atomic_inc(&dev->depth_left);
2025         /*
2026          * For SAM Task Attribute emulation for failed struct se_cmd
2027          */
2028         if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
2029                 transport_complete_task_attr(cmd);
2030
2031         if (complete) {
2032                 transport_direct_request_timeout(cmd);
2033                 cmd->transport_error_status = PYX_TRANSPORT_LU_COMM_FAILURE;
2034         }
2035
2036         switch (cmd->transport_error_status) {
2037         case PYX_TRANSPORT_UNKNOWN_SAM_OPCODE:
2038                 cmd->scsi_sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
2039                 break;
2040         case PYX_TRANSPORT_REQ_TOO_MANY_SECTORS:
2041                 cmd->scsi_sense_reason = TCM_SECTOR_COUNT_TOO_MANY;
2042                 break;
2043         case PYX_TRANSPORT_INVALID_CDB_FIELD:
2044                 cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD;
2045                 break;
2046         case PYX_TRANSPORT_INVALID_PARAMETER_LIST:
2047                 cmd->scsi_sense_reason = TCM_INVALID_PARAMETER_LIST;
2048                 break;
2049         case PYX_TRANSPORT_OUT_OF_MEMORY_RESOURCES:
2050                 if (!sc)
2051                         transport_new_cmd_failure(cmd);
2052                 /*
2053                  * Currently for PYX_TRANSPORT_OUT_OF_MEMORY_RESOURCES,
2054                  * we force this session to fall back to session
2055                  * recovery.
2056                  */
2057                 cmd->se_tfo->fall_back_to_erl0(cmd->se_sess);
2058                 cmd->se_tfo->stop_session(cmd->se_sess, 0, 0);
2059
2060                 goto check_stop;
2061         case PYX_TRANSPORT_LU_COMM_FAILURE:
2062         case PYX_TRANSPORT_ILLEGAL_REQUEST:
2063                 cmd->scsi_sense_reason = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
2064                 break;
2065         case PYX_TRANSPORT_UNKNOWN_MODE_PAGE:
2066                 cmd->scsi_sense_reason = TCM_UNKNOWN_MODE_PAGE;
2067                 break;
2068         case PYX_TRANSPORT_WRITE_PROTECTED:
2069                 cmd->scsi_sense_reason = TCM_WRITE_PROTECTED;
2070                 break;
2071         case PYX_TRANSPORT_RESERVATION_CONFLICT:
2072                 /*
2073                  * No SENSE Data payload for this case, set SCSI Status
2074                  * and queue the response to $FABRIC_MOD.
2075                  *
2076                  * Uses linux/include/scsi/scsi.h SAM status codes defs
2077                  */
2078                 cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
2079                 /*
2080                  * For UA Interlock Code 11b, a RESERVATION CONFLICT will
2081                  * establish a UNIT ATTENTION with PREVIOUS RESERVATION
2082                  * CONFLICT STATUS.
2083                  *
2084                  * See spc4r17, section 7.4.6 Control Mode Page, Table 349
2085                  */
2086                 if (cmd->se_sess &&
2087                     cmd->se_dev->se_sub_dev->se_dev_attrib.emulate_ua_intlck_ctrl == 2)
2088                         core_scsi3_ua_allocate(cmd->se_sess->se_node_acl,
2089                                 cmd->orig_fe_lun, 0x2C,
2090                                 ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS);
2091
2092                 cmd->se_tfo->queue_status(cmd);
2093                 goto check_stop;
2094         case PYX_TRANSPORT_USE_SENSE_REASON:
2095                 /*
2096                  * struct se_cmd->scsi_sense_reason already set
2097                  */
2098                 break;
2099         default:
2100                 printk(KERN_ERR "Unknown transport error for CDB 0x%02x: %d\n",
2101                         cmd->t_task.t_task_cdb[0],
2102                         cmd->transport_error_status);
2103                 cmd->scsi_sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
2104                 break;
2105         }
2106
2107         if (!sc)
2108                 transport_new_cmd_failure(cmd);
2109         else
2110                 transport_send_check_condition_and_sense(cmd,
2111                         cmd->scsi_sense_reason, 0);
2112 check_stop:
2113         transport_lun_remove_cmd(cmd);
2114         if (!(transport_cmd_check_stop_to_fabric(cmd)))
2115                 ;
2116 }
2117
2118 static void transport_direct_request_timeout(struct se_cmd *cmd)
2119 {
2120         unsigned long flags;
2121
2122         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
2123         if (!(atomic_read(&cmd->t_task.t_transport_timeout))) {
2124                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
2125                 return;
2126         }
2127         if (atomic_read(&cmd->t_task.t_task_cdbs_timeout_left)) {
2128                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
2129                 return;
2130         }
2131
2132         atomic_sub(atomic_read(&cmd->t_task.t_transport_timeout),
2133                    &cmd->t_task.t_se_count);
2134         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
2135 }
2136
2137 static void transport_generic_request_timeout(struct se_cmd *cmd)
2138 {
2139         unsigned long flags;
2140
2141         /*
2142          * Reset cmd->t_task.t_se_count to allow transport_generic_remove()
2143          * to allow last call to free memory resources.
2144          */
2145         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
2146         if (atomic_read(&cmd->t_task.t_transport_timeout) > 1) {
2147                 int tmp = (atomic_read(&cmd->t_task.t_transport_timeout) - 1);
2148
2149                 atomic_sub(tmp, &cmd->t_task.t_se_count);
2150         }
2151         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
2152
2153         transport_generic_remove(cmd, 0, 0);
2154 }
2155
2156 static int
2157 transport_generic_allocate_buf(struct se_cmd *cmd, u32 data_length)
2158 {
2159         unsigned char *buf;
2160
2161         buf = kzalloc(data_length, GFP_KERNEL);
2162         if (!(buf)) {
2163                 printk(KERN_ERR "Unable to allocate memory for buffer\n");
2164                 return -ENOMEM;
2165         }
2166
2167         cmd->t_task.t_tasks_se_num = 0;
2168         cmd->t_task.t_task_buf = buf;
2169
2170         return 0;
2171 }
2172
2173 static inline u32 transport_lba_21(unsigned char *cdb)
2174 {
2175         return ((cdb[1] & 0x1f) << 16) | (cdb[2] << 8) | cdb[3];
2176 }
2177
2178 static inline u32 transport_lba_32(unsigned char *cdb)
2179 {
2180         return (cdb[2] << 24) | (cdb[3] << 16) | (cdb[4] << 8) | cdb[5];
2181 }
2182
2183 static inline unsigned long long transport_lba_64(unsigned char *cdb)
2184 {
2185         unsigned int __v1, __v2;
2186
2187         __v1 = (cdb[2] << 24) | (cdb[3] << 16) | (cdb[4] << 8) | cdb[5];
2188         __v2 = (cdb[6] << 24) | (cdb[7] << 16) | (cdb[8] << 8) | cdb[9];
2189
2190         return ((unsigned long long)__v2) | (unsigned long long)__v1 << 32;
2191 }
2192
2193 /*
2194  * For VARIABLE_LENGTH_CDB w/ 32 byte extended CDBs
2195  */
2196 static inline unsigned long long transport_lba_64_ext(unsigned char *cdb)
2197 {
2198         unsigned int __v1, __v2;
2199
2200         __v1 = (cdb[12] << 24) | (cdb[13] << 16) | (cdb[14] << 8) | cdb[15];
2201         __v2 = (cdb[16] << 24) | (cdb[17] << 16) | (cdb[18] << 8) | cdb[19];
2202
2203         return ((unsigned long long)__v2) | (unsigned long long)__v1 << 32;
2204 }
2205
2206 static void transport_set_supported_SAM_opcode(struct se_cmd *se_cmd)
2207 {
2208         unsigned long flags;
2209
2210         spin_lock_irqsave(&se_cmd->t_task.t_state_lock, flags);
2211         se_cmd->se_cmd_flags |= SCF_SUPPORTED_SAM_OPCODE;
2212         spin_unlock_irqrestore(&se_cmd->t_task.t_state_lock, flags);
2213 }
2214
2215 /*
2216  * Called from interrupt context.
2217  */
2218 static void transport_task_timeout_handler(unsigned long data)
2219 {
2220         struct se_task *task = (struct se_task *)data;
2221         struct se_cmd *cmd = task->task_se_cmd;
2222         unsigned long flags;
2223
2224         DEBUG_TT("transport task timeout fired! task: %p cmd: %p\n", task, cmd);
2225
2226         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
2227         if (task->task_flags & TF_STOP) {
2228                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
2229                 return;
2230         }
2231         task->task_flags &= ~TF_RUNNING;
2232
2233         /*
2234          * Determine if transport_complete_task() has already been called.
2235          */
2236         if (!(atomic_read(&task->task_active))) {
2237                 DEBUG_TT("transport task: %p cmd: %p timeout task_active"
2238                                 " == 0\n", task, cmd);
2239                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
2240                 return;
2241         }
2242
2243         atomic_inc(&cmd->t_task.t_se_count);
2244         atomic_inc(&cmd->t_task.t_transport_timeout);
2245         cmd->t_task.t_tasks_failed = 1;
2246
2247         atomic_set(&task->task_timeout, 1);
2248         task->task_error_status = PYX_TRANSPORT_TASK_TIMEOUT;
2249         task->task_scsi_status = 1;
2250
2251         if (atomic_read(&task->task_stop)) {
2252                 DEBUG_TT("transport task: %p cmd: %p timeout task_stop"
2253                                 " == 1\n", task, cmd);
2254                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
2255                 complete(&task->task_stop_comp);
2256                 return;
2257         }
2258
2259         if (!(atomic_dec_and_test(&cmd->t_task.t_task_cdbs_left))) {
2260                 DEBUG_TT("transport task: %p cmd: %p timeout non zero"
2261                                 " t_task_cdbs_left\n", task, cmd);
2262                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
2263                 return;
2264         }
2265         DEBUG_TT("transport task: %p cmd: %p timeout ZERO t_task_cdbs_left\n",
2266                         task, cmd);
2267
2268         cmd->t_state = TRANSPORT_COMPLETE_FAILURE;
2269         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
2270
2271         transport_add_cmd_to_queue(cmd, TRANSPORT_COMPLETE_FAILURE);
2272 }
2273
2274 /*
2275  * Called with cmd->t_task.t_state_lock held.
2276  */
2277 static void transport_start_task_timer(struct se_task *task)
2278 {
2279         struct se_device *dev = task->se_dev;
2280         int timeout;
2281
2282         if (task->task_flags & TF_RUNNING)
2283                 return;
2284         /*
2285          * If the task_timeout is disabled, exit now.
2286          */
2287         timeout = dev->se_sub_dev->se_dev_attrib.task_timeout;
2288         if (!(timeout))
2289                 return;
2290
2291         init_timer(&task->task_timer);
2292         task->task_timer.expires = (get_jiffies_64() + timeout * HZ);
2293         task->task_timer.data = (unsigned long) task;
2294         task->task_timer.function = transport_task_timeout_handler;
2295
2296         task->task_flags |= TF_RUNNING;
2297         add_timer(&task->task_timer);
2298 #if 0
2299         printk(KERN_INFO "Starting task timer for cmd: %p task: %p seconds:"
2300                 " %d\n", task->task_se_cmd, task, timeout);
2301 #endif
2302 }
2303
2304 /*
2305  * Called with spin_lock_irq(&cmd->t_task.t_state_lock) held.
2306  */
2307 void __transport_stop_task_timer(struct se_task *task, unsigned long *flags)
2308 {
2309         struct se_cmd *cmd = task->task_se_cmd;
2310
2311         if (!(task->task_flags & TF_RUNNING))
2312                 return;
2313
2314         task->task_flags |= TF_STOP;
2315         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, *flags);
2316
2317         del_timer_sync(&task->task_timer);
2318
2319         spin_lock_irqsave(&cmd->t_task.t_state_lock, *flags);
2320         task->task_flags &= ~TF_RUNNING;
2321         task->task_flags &= ~TF_STOP;
2322 }
2323
2324 static void transport_stop_all_task_timers(struct se_cmd *cmd)
2325 {
2326         struct se_task *task = NULL, *task_tmp;
2327         unsigned long flags;
2328
2329         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
2330         list_for_each_entry_safe(task, task_tmp,
2331                                 &cmd->t_task.t_task_list, t_list)
2332                 __transport_stop_task_timer(task, &flags);
2333         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
2334 }
2335
2336 static inline int transport_tcq_window_closed(struct se_device *dev)
2337 {
2338         if (dev->dev_tcq_window_closed++ <
2339                         PYX_TRANSPORT_WINDOW_CLOSED_THRESHOLD) {
2340                 msleep(PYX_TRANSPORT_WINDOW_CLOSED_WAIT_SHORT);
2341         } else
2342                 msleep(PYX_TRANSPORT_WINDOW_CLOSED_WAIT_LONG);
2343
2344         wake_up_interruptible(&dev->dev_queue_obj.thread_wq);
2345         return 0;
2346 }
2347
2348 /*
2349  * Called from Fabric Module context from transport_execute_tasks()
2350  *
2351  * The return of this function determins if the tasks from struct se_cmd
2352  * get added to the execution queue in transport_execute_tasks(),
2353  * or are added to the delayed or ordered lists here.
2354  */
2355 static inline int transport_execute_task_attr(struct se_cmd *cmd)
2356 {
2357         if (cmd->se_dev->dev_task_attr_type != SAM_TASK_ATTR_EMULATED)
2358                 return 1;
2359         /*
2360          * Check for the existence of HEAD_OF_QUEUE, and if true return 1
2361          * to allow the passed struct se_cmd list of tasks to the front of the list.
2362          */
2363          if (cmd->sam_task_attr == MSG_HEAD_TAG) {
2364                 atomic_inc(&cmd->se_dev->dev_hoq_count);
2365                 smp_mb__after_atomic_inc();
2366                 DEBUG_STA("Added HEAD_OF_QUEUE for CDB:"
2367                         " 0x%02x, se_ordered_id: %u\n",
2368                         cmd->t_task->t_task_cdb[0],
2369                         cmd->se_ordered_id);
2370                 return 1;
2371         } else if (cmd->sam_task_attr == MSG_ORDERED_TAG) {
2372                 spin_lock(&cmd->se_dev->ordered_cmd_lock);
2373                 list_add_tail(&cmd->se_ordered_node,
2374                                 &cmd->se_dev->ordered_cmd_list);
2375                 spin_unlock(&cmd->se_dev->ordered_cmd_lock);
2376
2377                 atomic_inc(&cmd->se_dev->dev_ordered_sync);
2378                 smp_mb__after_atomic_inc();
2379
2380                 DEBUG_STA("Added ORDERED for CDB: 0x%02x to ordered"
2381                                 " list, se_ordered_id: %u\n",
2382                                 cmd->t_task.t_task_cdb[0],
2383                                 cmd->se_ordered_id);
2384                 /*
2385                  * Add ORDERED command to tail of execution queue if
2386                  * no other older commands exist that need to be
2387                  * completed first.
2388                  */
2389                 if (!(atomic_read(&cmd->se_dev->simple_cmds)))
2390                         return 1;
2391         } else {
2392                 /*
2393                  * For SIMPLE and UNTAGGED Task Attribute commands
2394                  */
2395                 atomic_inc(&cmd->se_dev->simple_cmds);
2396                 smp_mb__after_atomic_inc();
2397         }
2398         /*
2399          * Otherwise if one or more outstanding ORDERED task attribute exist,
2400          * add the dormant task(s) built for the passed struct se_cmd to the
2401          * execution queue and become in Active state for this struct se_device.
2402          */
2403         if (atomic_read(&cmd->se_dev->dev_ordered_sync) != 0) {
2404                 /*
2405                  * Otherwise, add cmd w/ tasks to delayed cmd queue that
2406                  * will be drained upon completion of HEAD_OF_QUEUE task.
2407                  */
2408                 spin_lock(&cmd->se_dev->delayed_cmd_lock);
2409                 cmd->se_cmd_flags |= SCF_DELAYED_CMD_FROM_SAM_ATTR;
2410                 list_add_tail(&cmd->se_delayed_node,
2411                                 &cmd->se_dev->delayed_cmd_list);
2412                 spin_unlock(&cmd->se_dev->delayed_cmd_lock);
2413
2414                 DEBUG_STA("Added CDB: 0x%02x Task Attr: 0x%02x to"
2415                         " delayed CMD list, se_ordered_id: %u\n",
2416                         cmd->t_task.t_task_cdb[0], cmd->sam_task_attr,
2417                         cmd->se_ordered_id);
2418                 /*
2419                  * Return zero to let transport_execute_tasks() know
2420                  * not to add the delayed tasks to the execution list.
2421                  */
2422                 return 0;
2423         }
2424         /*
2425          * Otherwise, no ORDERED task attributes exist..
2426          */
2427         return 1;
2428 }
2429
2430 /*
2431  * Called from fabric module context in transport_generic_new_cmd() and
2432  * transport_generic_process_write()
2433  */
2434 static int transport_execute_tasks(struct se_cmd *cmd)
2435 {
2436         int add_tasks;
2437
2438         if (!(cmd->se_cmd_flags & SCF_SE_DISABLE_ONLINE_CHECK)) {
2439                 if (se_dev_check_online(cmd->se_orig_obj_ptr) != 0) {
2440                         cmd->transport_error_status =
2441                                 PYX_TRANSPORT_LU_COMM_FAILURE;
2442                         transport_generic_request_failure(cmd, NULL, 0, 1);
2443                         return 0;
2444                 }
2445         }
2446         /*
2447          * Call transport_cmd_check_stop() to see if a fabric exception
2448          * has occurred that prevents execution.
2449          */
2450         if (!(transport_cmd_check_stop(cmd, 0, TRANSPORT_PROCESSING))) {
2451                 /*
2452                  * Check for SAM Task Attribute emulation and HEAD_OF_QUEUE
2453                  * attribute for the tasks of the received struct se_cmd CDB
2454                  */
2455                 add_tasks = transport_execute_task_attr(cmd);
2456                 if (!add_tasks)
2457                         goto execute_tasks;
2458                 /*
2459                  * This calls transport_add_tasks_from_cmd() to handle
2460                  * HEAD_OF_QUEUE ordering for SAM Task Attribute emulation
2461                  * (if enabled) in __transport_add_task_to_execute_queue() and
2462                  * transport_add_task_check_sam_attr().
2463                  */
2464                 transport_add_tasks_from_cmd(cmd);
2465         }
2466         /*
2467          * Kick the execution queue for the cmd associated struct se_device
2468          * storage object.
2469          */
2470 execute_tasks:
2471         __transport_execute_tasks(cmd->se_dev);
2472         return 0;
2473 }
2474
2475 /*
2476  * Called to check struct se_device tcq depth window, and once open pull struct se_task
2477  * from struct se_device->execute_task_list and
2478  *
2479  * Called from transport_processing_thread()
2480  */
2481 static int __transport_execute_tasks(struct se_device *dev)
2482 {
2483         int error;
2484         struct se_cmd *cmd = NULL;
2485         struct se_task *task = NULL;
2486         unsigned long flags;
2487
2488         /*
2489          * Check if there is enough room in the device and HBA queue to send
2490          * struct se_transport_task's to the selected transport.
2491          */
2492 check_depth:
2493         if (!atomic_read(&dev->depth_left))
2494                 return transport_tcq_window_closed(dev);
2495
2496         dev->dev_tcq_window_closed = 0;
2497
2498         spin_lock_irq(&dev->execute_task_lock);
2499         if (list_empty(&dev->execute_task_list)) {
2500                 spin_unlock_irq(&dev->execute_task_lock);
2501                 return 0;
2502         }
2503         task = list_first_entry(&dev->execute_task_list,
2504                                 struct se_task, t_execute_list);
2505         list_del(&task->t_execute_list);
2506         atomic_set(&task->task_execute_queue, 0);
2507         atomic_dec(&dev->execute_tasks);
2508         spin_unlock_irq(&dev->execute_task_lock);
2509
2510         atomic_dec(&dev->depth_left);
2511
2512         cmd = task->task_se_cmd;
2513
2514         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
2515         atomic_set(&task->task_active, 1);
2516         atomic_set(&task->task_sent, 1);
2517         atomic_inc(&cmd->t_task.t_task_cdbs_sent);
2518
2519         if (atomic_read(&cmd->t_task.t_task_cdbs_sent) ==
2520             cmd->t_task.t_task_cdbs)
2521                 atomic_set(&cmd->transport_sent, 1);
2522
2523         transport_start_task_timer(task);
2524         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
2525         /*
2526          * The struct se_cmd->transport_emulate_cdb() function pointer is used
2527          * to grab REPORT_LUNS and other CDBs we want to handle before they hit the
2528          * struct se_subsystem_api->do_task() caller below.
2529          */
2530         if (cmd->transport_emulate_cdb) {
2531                 error = cmd->transport_emulate_cdb(cmd);
2532                 if (error != 0) {
2533                         cmd->transport_error_status = error;
2534                         atomic_set(&task->task_active, 0);
2535                         atomic_set(&cmd->transport_sent, 0);
2536                         transport_stop_tasks_for_cmd(cmd);
2537                         transport_generic_request_failure(cmd, dev, 0, 1);
2538                         goto check_depth;
2539                 }
2540                 /*
2541                  * Handle the successful completion for transport_emulate_cdb()
2542                  * for synchronous operation, following SCF_EMULATE_CDB_ASYNC
2543                  * Otherwise the caller is expected to complete the task with
2544                  * proper status.
2545                  */
2546                 if (!(cmd->se_cmd_flags & SCF_EMULATE_CDB_ASYNC)) {
2547                         cmd->scsi_status = SAM_STAT_GOOD;
2548                         task->task_scsi_status = GOOD;
2549                         transport_complete_task(task, 1);
2550                 }
2551         } else {
2552                 /*
2553                  * Currently for all virtual TCM plugins including IBLOCK, FILEIO and
2554                  * RAMDISK we use the internal transport_emulate_control_cdb() logic
2555                  * with struct se_subsystem_api callers for the primary SPC-3 TYPE_DISK
2556                  * LUN emulation code.
2557                  *
2558                  * For TCM/pSCSI and all other SCF_SCSI_DATA_SG_IO_CDB I/O tasks we
2559                  * call ->do_task() directly and let the underlying TCM subsystem plugin
2560                  * code handle the CDB emulation.
2561                  */
2562                 if ((dev->transport->transport_type != TRANSPORT_PLUGIN_PHBA_PDEV) &&
2563                     (!(task->task_se_cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB)))
2564                         error = transport_emulate_control_cdb(task);
2565                 else
2566                         error = dev->transport->do_task(task);
2567
2568                 if (error != 0) {
2569                         cmd->transport_error_status = error;
2570                         atomic_set(&task->task_active, 0);
2571                         atomic_set(&cmd->transport_sent, 0);
2572                         transport_stop_tasks_for_cmd(cmd);
2573                         transport_generic_request_failure(cmd, dev, 0, 1);
2574                 }
2575         }
2576
2577         goto check_depth;
2578
2579         return 0;
2580 }
2581
2582 void transport_new_cmd_failure(struct se_cmd *se_cmd)
2583 {
2584         unsigned long flags;
2585         /*
2586          * Any unsolicited data will get dumped for failed command inside of
2587          * the fabric plugin
2588          */
2589         spin_lock_irqsave(&se_cmd->t_task.t_state_lock, flags);
2590         se_cmd->se_cmd_flags |= SCF_SE_CMD_FAILED;
2591         se_cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
2592         spin_unlock_irqrestore(&se_cmd->t_task.t_state_lock, flags);
2593
2594         se_cmd->se_tfo->new_cmd_failure(se_cmd);
2595 }
2596
2597 static void transport_nop_wait_for_tasks(struct se_cmd *, int, int);
2598
2599 static inline u32 transport_get_sectors_6(
2600         unsigned char *cdb,
2601         struct se_cmd *cmd,
2602         int *ret)
2603 {
2604         struct se_device *dev = cmd->se_dev;
2605
2606         /*
2607          * Assume TYPE_DISK for non struct se_device objects.
2608          * Use 8-bit sector value.
2609          */
2610         if (!dev)
2611                 goto type_disk;
2612
2613         /*
2614          * Use 24-bit allocation length for TYPE_TAPE.
2615          */
2616         if (dev->transport->get_device_type(dev) == TYPE_TAPE)
2617                 return (u32)(cdb[2] << 16) + (cdb[3] << 8) + cdb[4];
2618
2619         /*
2620          * Everything else assume TYPE_DISK Sector CDB location.
2621          * Use 8-bit sector value.
2622          */
2623 type_disk:
2624         return (u32)cdb[4];
2625 }
2626
2627 static inline u32 transport_get_sectors_10(
2628         unsigned char *cdb,
2629         struct se_cmd *cmd,
2630         int *ret)
2631 {
2632         struct se_device *dev = cmd->se_dev;
2633
2634         /*
2635          * Assume TYPE_DISK for non struct se_device objects.
2636          * Use 16-bit sector value.
2637          */
2638         if (!dev)
2639                 goto type_disk;
2640
2641         /*
2642          * XXX_10 is not defined in SSC, throw an exception
2643          */
2644         if (dev->transport->get_device_type(dev) == TYPE_TAPE) {
2645                 *ret = -EINVAL;
2646                 return 0;
2647         }
2648
2649         /*
2650          * Everything else assume TYPE_DISK Sector CDB location.
2651          * Use 16-bit sector value.
2652          */
2653 type_disk:
2654         return (u32)(cdb[7] << 8) + cdb[8];
2655 }
2656
2657 static inline u32 transport_get_sectors_12(
2658         unsigned char *cdb,
2659         struct se_cmd *cmd,
2660         int *ret)
2661 {
2662         struct se_device *dev = cmd->se_dev;
2663
2664         /*
2665          * Assume TYPE_DISK for non struct se_device objects.
2666          * Use 32-bit sector value.
2667          */
2668         if (!dev)
2669                 goto type_disk;
2670
2671         /*
2672          * XXX_12 is not defined in SSC, throw an exception
2673          */
2674         if (dev->transport->get_device_type(dev) == TYPE_TAPE) {
2675                 *ret = -EINVAL;
2676                 return 0;
2677         }
2678
2679         /*
2680          * Everything else assume TYPE_DISK Sector CDB location.
2681          * Use 32-bit sector value.
2682          */
2683 type_disk:
2684         return (u32)(cdb[6] << 24) + (cdb[7] << 16) + (cdb[8] << 8) + cdb[9];
2685 }
2686
2687 static inline u32 transport_get_sectors_16(
2688         unsigned char *cdb,
2689         struct se_cmd *cmd,
2690         int *ret)
2691 {
2692         struct se_device *dev = cmd->se_dev;
2693
2694         /*
2695          * Assume TYPE_DISK for non struct se_device objects.
2696          * Use 32-bit sector value.
2697          */
2698         if (!dev)
2699                 goto type_disk;
2700
2701         /*
2702          * Use 24-bit allocation length for TYPE_TAPE.
2703          */
2704         if (dev->transport->get_device_type(dev) == TYPE_TAPE)
2705                 return (u32)(cdb[12] << 16) + (cdb[13] << 8) + cdb[14];
2706
2707 type_disk:
2708         return (u32)(cdb[10] << 24) + (cdb[11] << 16) +
2709                     (cdb[12] << 8) + cdb[13];
2710 }
2711
2712 /*
2713  * Used for VARIABLE_LENGTH_CDB WRITE_32 and READ_32 variants
2714  */
2715 static inline u32 transport_get_sectors_32(
2716         unsigned char *cdb,
2717         struct se_cmd *cmd,
2718         int *ret)
2719 {
2720         /*
2721          * Assume TYPE_DISK for non struct se_device objects.
2722          * Use 32-bit sector value.
2723          */
2724         return (u32)(cdb[28] << 24) + (cdb[29] << 16) +
2725                     (cdb[30] << 8) + cdb[31];
2726
2727 }
2728
2729 static inline u32 transport_get_size(
2730         u32 sectors,
2731         unsigned char *cdb,
2732         struct se_cmd *cmd)
2733 {
2734         struct se_device *dev = cmd->se_dev;
2735
2736         if (dev->transport->get_device_type(dev) == TYPE_TAPE) {
2737                 if (cdb[1] & 1) { /* sectors */
2738                         return dev->se_sub_dev->se_dev_attrib.block_size * sectors;
2739                 } else /* bytes */
2740                         return sectors;
2741         }
2742 #if 0
2743         printk(KERN_INFO "Returning block_size: %u, sectors: %u == %u for"
2744                         " %s object\n", dev->se_sub_dev->se_dev_attrib.block_size, sectors,
2745                         dev->se_sub_dev->se_dev_attrib.block_size * sectors,
2746                         dev->transport->name);
2747 #endif
2748         return dev->se_sub_dev->se_dev_attrib.block_size * sectors;
2749 }
2750
2751 unsigned char transport_asciihex_to_binaryhex(unsigned char val[2])
2752 {
2753         unsigned char result = 0;
2754         /*
2755          * MSB
2756          */
2757         if ((val[0] >= 'a') && (val[0] <= 'f'))
2758                 result = ((val[0] - 'a' + 10) & 0xf) << 4;
2759         else
2760                 if ((val[0] >= 'A') && (val[0] <= 'F'))
2761                         result = ((val[0] - 'A' + 10) & 0xf) << 4;
2762                 else /* digit */
2763                         result = ((val[0] - '0') & 0xf) << 4;
2764         /*
2765          * LSB
2766          */
2767         if ((val[1] >= 'a') && (val[1] <= 'f'))
2768                 result |= ((val[1] - 'a' + 10) & 0xf);
2769         else
2770                 if ((val[1] >= 'A') && (val[1] <= 'F'))
2771                         result |= ((val[1] - 'A' + 10) & 0xf);
2772                 else /* digit */
2773                         result |= ((val[1] - '0') & 0xf);
2774
2775         return result;
2776 }
2777 EXPORT_SYMBOL(transport_asciihex_to_binaryhex);
2778
2779 static void transport_xor_callback(struct se_cmd *cmd)
2780 {
2781         unsigned char *buf, *addr;
2782         struct se_mem *se_mem;
2783         unsigned int offset;
2784         int i;
2785         /*
2786          * From sbc3r22.pdf section 5.48 XDWRITEREAD (10) command
2787          *
2788          * 1) read the specified logical block(s);
2789          * 2) transfer logical blocks from the data-out buffer;
2790          * 3) XOR the logical blocks transferred from the data-out buffer with
2791          *    the logical blocks read, storing the resulting XOR data in a buffer;
2792          * 4) if the DISABLE WRITE bit is set to zero, then write the logical
2793          *    blocks transferred from the data-out buffer; and
2794          * 5) transfer the resulting XOR data to the data-in buffer.
2795          */
2796         buf = kmalloc(cmd->data_length, GFP_KERNEL);
2797         if (!(buf)) {
2798                 printk(KERN_ERR "Unable to allocate xor_callback buf\n");
2799                 return;
2800         }
2801         /*
2802          * Copy the scatterlist WRITE buffer located at cmd->t_task.t_mem_list
2803          * into the locally allocated *buf
2804          */
2805         transport_memcpy_se_mem_read_contig(cmd, buf, &cmd->t_task.t_mem_list);
2806         /*
2807          * Now perform the XOR against the BIDI read memory located at
2808          * cmd->t_task.t_mem_bidi_list
2809          */
2810
2811         offset = 0;
2812         list_for_each_entry(se_mem, &cmd->t_task.t_mem_bidi_list, se_list) {
2813                 addr = (unsigned char *)kmap_atomic(se_mem->se_page, KM_USER0);
2814                 if (!(addr))
2815                         goto out;
2816
2817                 for (i = 0; i < se_mem->se_len; i++)
2818                         *(addr + se_mem->se_off + i) ^= *(buf + offset + i);
2819
2820                 offset += se_mem->se_len;
2821                 kunmap_atomic(addr, KM_USER0);
2822         }
2823 out:
2824         kfree(buf);
2825 }
2826
2827 /*
2828  * Used to obtain Sense Data from underlying Linux/SCSI struct scsi_cmnd
2829  */
2830 static int transport_get_sense_data(struct se_cmd *cmd)
2831 {
2832         unsigned char *buffer = cmd->sense_buffer, *sense_buffer = NULL;
2833         struct se_device *dev;
2834         struct se_task *task = NULL, *task_tmp;
2835         unsigned long flags;
2836         u32 offset = 0;
2837
2838         WARN_ON(!cmd->se_lun);
2839
2840         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
2841         if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
2842                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
2843                 return 0;
2844         }
2845
2846         list_for_each_entry_safe(task, task_tmp,
2847                                 &cmd->t_task.t_task_list, t_list) {
2848
2849                 if (!task->task_sense)
2850                         continue;
2851
2852                 dev = task->se_dev;
2853                 if (!(dev))
2854                         continue;
2855
2856                 if (!dev->transport->get_sense_buffer) {
2857                         printk(KERN_ERR "dev->transport->get_sense_buffer"
2858                                         " is NULL\n");
2859                         continue;
2860                 }
2861
2862                 sense_buffer = dev->transport->get_sense_buffer(task);
2863                 if (!(sense_buffer)) {
2864                         printk(KERN_ERR "ITT[0x%08x]_TASK[%d]: Unable to locate"
2865                                 " sense buffer for task with sense\n",
2866                                 cmd->se_tfo->get_task_tag(cmd), task->task_no);
2867                         continue;
2868                 }
2869                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
2870
2871                 offset = cmd->se_tfo->set_fabric_sense_len(cmd,
2872                                 TRANSPORT_SENSE_BUFFER);
2873
2874                 memcpy(&buffer[offset], sense_buffer,
2875                                 TRANSPORT_SENSE_BUFFER);
2876                 cmd->scsi_status = task->task_scsi_status;
2877                 /* Automatically padded */
2878                 cmd->scsi_sense_length =
2879                                 (TRANSPORT_SENSE_BUFFER + offset);
2880
2881                 printk(KERN_INFO "HBA_[%u]_PLUG[%s]: Set SAM STATUS: 0x%02x"
2882                                 " and sense\n",
2883                         dev->se_hba->hba_id, dev->transport->name,
2884                                 cmd->scsi_status);
2885                 return 0;
2886         }
2887         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
2888
2889         return -1;
2890 }
2891
2892 static int transport_allocate_resources(struct se_cmd *cmd)
2893 {
2894         u32 length = cmd->data_length;
2895
2896         if ((cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB) ||
2897             (cmd->se_cmd_flags & SCF_SCSI_CONTROL_SG_IO_CDB))
2898                 return transport_generic_get_mem(cmd, length, PAGE_SIZE);
2899         else if (cmd->se_cmd_flags & SCF_SCSI_CONTROL_NONSG_IO_CDB)
2900                 return transport_generic_allocate_buf(cmd, length);
2901         else
2902                 return 0;
2903 }
2904
2905 static int
2906 transport_handle_reservation_conflict(struct se_cmd *cmd)
2907 {
2908         cmd->transport_wait_for_tasks = &transport_nop_wait_for_tasks;
2909         cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
2910         cmd->se_cmd_flags |= SCF_SCSI_RESERVATION_CONFLICT;
2911         cmd->scsi_status = SAM_STAT_RESERVATION_CONFLICT;
2912         /*
2913          * For UA Interlock Code 11b, a RESERVATION CONFLICT will
2914          * establish a UNIT ATTENTION with PREVIOUS RESERVATION
2915          * CONFLICT STATUS.
2916          *
2917          * See spc4r17, section 7.4.6 Control Mode Page, Table 349
2918          */
2919         if (cmd->se_sess &&
2920             cmd->se_dev->se_sub_dev->se_dev_attrib.emulate_ua_intlck_ctrl == 2)
2921                 core_scsi3_ua_allocate(cmd->se_sess->se_node_acl,
2922                         cmd->orig_fe_lun, 0x2C,
2923                         ASCQ_2CH_PREVIOUS_RESERVATION_CONFLICT_STATUS);
2924         return -EINVAL;
2925 }
2926
2927 /*      transport_generic_cmd_sequencer():
2928  *
2929  *      Generic Command Sequencer that should work for most DAS transport
2930  *      drivers.
2931  *
2932  *      Called from transport_generic_allocate_tasks() in the $FABRIC_MOD
2933  *      RX Thread.
2934  *
2935  *      FIXME: Need to support other SCSI OPCODES where as well.
2936  */
2937 static int transport_generic_cmd_sequencer(
2938         struct se_cmd *cmd,
2939         unsigned char *cdb)
2940 {
2941         struct se_device *dev = cmd->se_dev;
2942         struct se_subsystem_dev *su_dev = dev->se_sub_dev;
2943         int ret = 0, sector_ret = 0, passthrough;
2944         u32 sectors = 0, size = 0, pr_reg_type = 0;
2945         u16 service_action;
2946         u8 alua_ascq = 0;
2947         /*
2948          * Check for an existing UNIT ATTENTION condition
2949          */
2950         if (core_scsi3_ua_check(cmd, cdb) < 0) {
2951                 cmd->transport_wait_for_tasks =
2952                                 &transport_nop_wait_for_tasks;
2953                 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
2954                 cmd->scsi_sense_reason = TCM_CHECK_CONDITION_UNIT_ATTENTION;
2955                 return -EINVAL;
2956         }
2957         /*
2958          * Check status of Asymmetric Logical Unit Assignment port
2959          */
2960         ret = su_dev->t10_alua.alua_state_check(cmd, cdb, &alua_ascq);
2961         if (ret != 0) {
2962                 cmd->transport_wait_for_tasks = &transport_nop_wait_for_tasks;
2963                 /*
2964                  * Set SCSI additional sense code (ASC) to 'LUN Not Accessible';
2965                  * The ALUA additional sense code qualifier (ASCQ) is determined
2966                  * by the ALUA primary or secondary access state..
2967                  */
2968                 if (ret > 0) {
2969 #if 0
2970                         printk(KERN_INFO "[%s]: ALUA TG Port not available,"
2971                                 " SenseKey: NOT_READY, ASC/ASCQ: 0x04/0x%02x\n",
2972                                 cmd->se_tfo->get_fabric_name(), alua_ascq);
2973 #endif
2974                         transport_set_sense_codes(cmd, 0x04, alua_ascq);
2975                         cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
2976                         cmd->scsi_sense_reason = TCM_CHECK_CONDITION_NOT_READY;
2977                         return -EINVAL;
2978                 }
2979                 goto out_invalid_cdb_field;
2980         }
2981         /*
2982          * Check status for SPC-3 Persistent Reservations
2983          */
2984         if (su_dev->t10_pr.pr_ops.t10_reservation_check(cmd, &pr_reg_type) != 0) {
2985                 if (su_dev->t10_pr.pr_ops.t10_seq_non_holder(
2986                                         cmd, cdb, pr_reg_type) != 0)
2987                         return transport_handle_reservation_conflict(cmd);
2988                 /*
2989                  * This means the CDB is allowed for the SCSI Initiator port
2990                  * when said port is *NOT* holding the legacy SPC-2 or
2991                  * SPC-3 Persistent Reservation.
2992                  */
2993         }
2994
2995         switch (cdb[0]) {
2996         case READ_6:
2997                 sectors = transport_get_sectors_6(cdb, cmd, &sector_ret);
2998                 if (sector_ret)
2999                         goto out_unsupported_cdb;
3000                 size = transport_get_size(sectors, cdb, cmd);
3001                 cmd->transport_split_cdb = &split_cdb_XX_6;
3002                 cmd->t_task.t_task_lba = transport_lba_21(cdb);
3003                 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3004                 break;
3005         case READ_10:
3006                 sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
3007                 if (sector_ret)
3008                         goto out_unsupported_cdb;
3009                 size = transport_get_size(sectors, cdb, cmd);
3010                 cmd->transport_split_cdb = &split_cdb_XX_10;
3011                 cmd->t_task.t_task_lba = transport_lba_32(cdb);
3012                 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3013                 break;
3014         case READ_12:
3015                 sectors = transport_get_sectors_12(cdb, cmd, &sector_ret);
3016                 if (sector_ret)
3017                         goto out_unsupported_cdb;
3018                 size = transport_get_size(sectors, cdb, cmd);
3019                 cmd->transport_split_cdb = &split_cdb_XX_12;
3020                 cmd->t_task.t_task_lba = transport_lba_32(cdb);
3021                 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3022                 break;
3023         case READ_16:
3024                 sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
3025                 if (sector_ret)
3026                         goto out_unsupported_cdb;
3027                 size = transport_get_size(sectors, cdb, cmd);
3028                 cmd->transport_split_cdb = &split_cdb_XX_16;
3029                 cmd->t_task.t_task_lba = transport_lba_64(cdb);
3030                 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3031                 break;
3032         case WRITE_6:
3033                 sectors = transport_get_sectors_6(cdb, cmd, &sector_ret);
3034                 if (sector_ret)
3035                         goto out_unsupported_cdb;
3036                 size = transport_get_size(sectors, cdb, cmd);
3037                 cmd->transport_split_cdb = &split_cdb_XX_6;
3038                 cmd->t_task.t_task_lba = transport_lba_21(cdb);
3039                 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3040                 break;
3041         case WRITE_10:
3042                 sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
3043                 if (sector_ret)
3044                         goto out_unsupported_cdb;
3045                 size = transport_get_size(sectors, cdb, cmd);
3046                 cmd->transport_split_cdb = &split_cdb_XX_10;
3047                 cmd->t_task.t_task_lba = transport_lba_32(cdb);
3048                 cmd->t_task.t_tasks_fua = (cdb[1] & 0x8);
3049                 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3050                 break;
3051         case WRITE_12:
3052                 sectors = transport_get_sectors_12(cdb, cmd, &sector_ret);
3053                 if (sector_ret)
3054                         goto out_unsupported_cdb;
3055                 size = transport_get_size(sectors, cdb, cmd);
3056                 cmd->transport_split_cdb = &split_cdb_XX_12;
3057                 cmd->t_task.t_task_lba = transport_lba_32(cdb);
3058                 cmd->t_task.t_tasks_fua = (cdb[1] & 0x8);
3059                 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3060                 break;
3061         case WRITE_16:
3062                 sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
3063                 if (sector_ret)
3064                         goto out_unsupported_cdb;
3065                 size = transport_get_size(sectors, cdb, cmd);
3066                 cmd->transport_split_cdb = &split_cdb_XX_16;
3067                 cmd->t_task.t_task_lba = transport_lba_64(cdb);
3068                 cmd->t_task.t_tasks_fua = (cdb[1] & 0x8);
3069                 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3070                 break;
3071         case XDWRITEREAD_10:
3072                 if ((cmd->data_direction != DMA_TO_DEVICE) ||
3073                     !(cmd->t_task.t_tasks_bidi))
3074                         goto out_invalid_cdb_field;
3075                 sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
3076                 if (sector_ret)
3077                         goto out_unsupported_cdb;
3078                 size = transport_get_size(sectors, cdb, cmd);
3079                 cmd->transport_split_cdb = &split_cdb_XX_10;
3080                 cmd->t_task.t_task_lba = transport_lba_32(cdb);
3081                 cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3082                 passthrough = (dev->transport->transport_type ==
3083                                 TRANSPORT_PLUGIN_PHBA_PDEV);
3084                 /*
3085                  * Skip the remaining assignments for TCM/PSCSI passthrough
3086                  */
3087                 if (passthrough)
3088                         break;
3089                 /*
3090                  * Setup BIDI XOR callback to be run during transport_generic_complete_ok()
3091                  */
3092                 cmd->transport_complete_callback = &transport_xor_callback;
3093                 cmd->t_task.t_tasks_fua = (cdb[1] & 0x8);
3094                 break;
3095         case VARIABLE_LENGTH_CMD:
3096                 service_action = get_unaligned_be16(&cdb[8]);
3097                 /*
3098                  * Determine if this is TCM/PSCSI device and we should disable
3099                  * internal emulation for this CDB.
3100                  */
3101                 passthrough = (dev->transport->transport_type ==
3102                                         TRANSPORT_PLUGIN_PHBA_PDEV);
3103
3104                 switch (service_action) {
3105                 case XDWRITEREAD_32:
3106                         sectors = transport_get_sectors_32(cdb, cmd, &sector_ret);
3107                         if (sector_ret)
3108                                 goto out_unsupported_cdb;
3109                         size = transport_get_size(sectors, cdb, cmd);
3110                         /*
3111                          * Use WRITE_32 and READ_32 opcodes for the emulated
3112                          * XDWRITE_READ_32 logic.
3113                          */
3114                         cmd->transport_split_cdb = &split_cdb_XX_32;
3115                         cmd->t_task.t_task_lba = transport_lba_64_ext(cdb);
3116                         cmd->se_cmd_flags |= SCF_SCSI_DATA_SG_IO_CDB;
3117
3118                         /*
3119                          * Skip the remaining assignments for TCM/PSCSI passthrough
3120                          */
3121                         if (passthrough)
3122                                 break;
3123
3124                         /*
3125                          * Setup BIDI XOR callback to be run during
3126                          * transport_generic_complete_ok()
3127                          */
3128                         cmd->transport_complete_callback = &transport_xor_callback;
3129                         cmd->t_task.t_tasks_fua = (cdb[10] & 0x8);
3130                         break;
3131                 case WRITE_SAME_32:
3132                         sectors = transport_get_sectors_32(cdb, cmd, &sector_ret);
3133                         if (sector_ret)
3134                                 goto out_unsupported_cdb;
3135                         size = transport_get_size(sectors, cdb, cmd);
3136                         cmd->t_task.t_task_lba = get_unaligned_be64(&cdb[12]);
3137                         cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3138
3139                         /*
3140                          * Skip the remaining assignments for TCM/PSCSI passthrough
3141                          */
3142                         if (passthrough)
3143                                 break;
3144
3145                         if ((cdb[10] & 0x04) || (cdb[10] & 0x02)) {
3146                                 printk(KERN_ERR "WRITE_SAME PBDATA and LBDATA"
3147                                         " bits not supported for Block Discard"
3148                                         " Emulation\n");
3149                                 goto out_invalid_cdb_field;
3150                         }
3151                         /*
3152                          * Currently for the emulated case we only accept
3153                          * tpws with the UNMAP=1 bit set.
3154                          */
3155                         if (!(cdb[10] & 0x08)) {
3156                                 printk(KERN_ERR "WRITE_SAME w/o UNMAP bit not"
3157                                         " supported for Block Discard Emulation\n");
3158                                 goto out_invalid_cdb_field;
3159                         }
3160                         break;
3161                 default:
3162                         printk(KERN_ERR "VARIABLE_LENGTH_CMD service action"
3163                                 " 0x%04x not supported\n", service_action);
3164                         goto out_unsupported_cdb;
3165                 }
3166                 break;
3167         case MAINTENANCE_IN:
3168                 if (dev->transport->get_device_type(dev) != TYPE_ROM) {
3169                         /* MAINTENANCE_IN from SCC-2 */
3170                         /*
3171                          * Check for emulated MI_REPORT_TARGET_PGS.
3172                          */
3173                         if (cdb[1] == MI_REPORT_TARGET_PGS) {
3174                                 cmd->transport_emulate_cdb =
3175                                 (su_dev->t10_alua.alua_type ==
3176                                  SPC3_ALUA_EMULATED) ?
3177                                 core_emulate_report_target_port_groups :
3178                                 NULL;
3179                         }
3180                         size = (cdb[6] << 24) | (cdb[7] << 16) |
3181                                (cdb[8] << 8) | cdb[9];
3182                 } else {
3183                         /* GPCMD_SEND_KEY from multi media commands */
3184                         size = (cdb[8] << 8) + cdb[9];
3185                 }
3186                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3187                 break;
3188         case MODE_SELECT:
3189                 size = cdb[4];
3190                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3191                 break;
3192         case MODE_SELECT_10:
3193                 size = (cdb[7] << 8) + cdb[8];
3194                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3195                 break;
3196         case MODE_SENSE:
3197                 size = cdb[4];
3198                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3199                 break;
3200         case MODE_SENSE_10:
3201         case GPCMD_READ_BUFFER_CAPACITY:
3202         case GPCMD_SEND_OPC:
3203         case LOG_SELECT:
3204         case LOG_SENSE:
3205                 size = (cdb[7] << 8) + cdb[8];
3206                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3207                 break;
3208         case READ_BLOCK_LIMITS:
3209                 size = READ_BLOCK_LEN;
3210                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3211                 break;
3212         case GPCMD_GET_CONFIGURATION:
3213         case GPCMD_READ_FORMAT_CAPACITIES:
3214         case GPCMD_READ_DISC_INFO:
3215         case GPCMD_READ_TRACK_RZONE_INFO:
3216                 size = (cdb[7] << 8) + cdb[8];
3217                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3218                 break;
3219         case PERSISTENT_RESERVE_IN:
3220         case PERSISTENT_RESERVE_OUT:
3221                 cmd->transport_emulate_cdb =
3222                         (su_dev->t10_pr.res_type ==
3223                          SPC3_PERSISTENT_RESERVATIONS) ?
3224                         core_scsi3_emulate_pr : NULL;
3225                 size = (cdb[7] << 8) + cdb[8];
3226                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3227                 break;
3228         case GPCMD_MECHANISM_STATUS:
3229         case GPCMD_READ_DVD_STRUCTURE:
3230                 size = (cdb[8] << 8) + cdb[9];
3231                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3232                 break;
3233         case READ_POSITION:
3234                 size = READ_POSITION_LEN;
3235                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3236                 break;
3237         case MAINTENANCE_OUT:
3238                 if (dev->transport->get_device_type(dev) != TYPE_ROM) {
3239                         /* MAINTENANCE_OUT from SCC-2
3240                          *
3241                          * Check for emulated MO_SET_TARGET_PGS.
3242                          */
3243                         if (cdb[1] == MO_SET_TARGET_PGS) {
3244                                 cmd->transport_emulate_cdb =
3245                                 (su_dev->t10_alua.alua_type ==
3246                                         SPC3_ALUA_EMULATED) ?
3247                                 core_emulate_set_target_port_groups :
3248                                 NULL;
3249                         }
3250
3251                         size = (cdb[6] << 24) | (cdb[7] << 16) |
3252                                (cdb[8] << 8) | cdb[9];
3253                 } else  {
3254                         /* GPCMD_REPORT_KEY from multi media commands */
3255                         size = (cdb[8] << 8) + cdb[9];
3256                 }
3257                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3258                 break;
3259         case INQUIRY:
3260                 size = (cdb[3] << 8) + cdb[4];
3261                 /*
3262                  * Do implict HEAD_OF_QUEUE processing for INQUIRY.
3263                  * See spc4r17 section 5.3
3264                  */
3265                 if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
3266                         cmd->sam_task_attr = MSG_HEAD_TAG;
3267                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3268                 break;
3269         case READ_BUFFER:
3270                 size = (cdb[6] << 16) + (cdb[7] << 8) + cdb[8];
3271                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3272                 break;
3273         case READ_CAPACITY:
3274                 size = READ_CAP_LEN;
3275                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3276                 break;
3277         case READ_MEDIA_SERIAL_NUMBER:
3278         case SECURITY_PROTOCOL_IN:
3279         case SECURITY_PROTOCOL_OUT:
3280                 size = (cdb[6] << 24) | (cdb[7] << 16) | (cdb[8] << 8) | cdb[9];
3281                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3282                 break;
3283         case SERVICE_ACTION_IN:
3284         case ACCESS_CONTROL_IN:
3285         case ACCESS_CONTROL_OUT:
3286         case EXTENDED_COPY:
3287         case READ_ATTRIBUTE:
3288         case RECEIVE_COPY_RESULTS:
3289         case WRITE_ATTRIBUTE:
3290                 size = (cdb[10] << 24) | (cdb[11] << 16) |
3291                        (cdb[12] << 8) | cdb[13];
3292                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3293                 break;
3294         case RECEIVE_DIAGNOSTIC:
3295         case SEND_DIAGNOSTIC:
3296                 size = (cdb[3] << 8) | cdb[4];
3297                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3298                 break;
3299 /* #warning FIXME: Figure out correct GPCMD_READ_CD blocksize. */
3300 #if 0
3301         case GPCMD_READ_CD:
3302                 sectors = (cdb[6] << 16) + (cdb[7] << 8) + cdb[8];
3303                 size = (2336 * sectors);
3304                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3305                 break;
3306 #endif
3307         case READ_TOC:
3308                 size = cdb[8];
3309                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3310                 break;
3311         case REQUEST_SENSE:
3312                 size = cdb[4];
3313                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3314                 break;
3315         case READ_ELEMENT_STATUS:
3316                 size = 65536 * cdb[7] + 256 * cdb[8] + cdb[9];
3317                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3318                 break;
3319         case WRITE_BUFFER:
3320                 size = (cdb[6] << 16) + (cdb[7] << 8) + cdb[8];
3321                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3322                 break;
3323         case RESERVE:
3324         case RESERVE_10:
3325                 /*
3326                  * The SPC-2 RESERVE does not contain a size in the SCSI CDB.
3327                  * Assume the passthrough or $FABRIC_MOD will tell us about it.
3328                  */
3329                 if (cdb[0] == RESERVE_10)
3330                         size = (cdb[7] << 8) | cdb[8];
3331                 else
3332                         size = cmd->data_length;
3333
3334                 /*
3335                  * Setup the legacy emulated handler for SPC-2 and
3336                  * >= SPC-3 compatible reservation handling (CRH=1)
3337                  * Otherwise, we assume the underlying SCSI logic is
3338                  * is running in SPC_PASSTHROUGH, and wants reservations
3339                  * emulation disabled.
3340                  */
3341                 cmd->transport_emulate_cdb =
3342                                 (su_dev->t10_pr.res_type !=
3343                                  SPC_PASSTHROUGH) ?
3344                                 core_scsi2_emulate_crh : NULL;
3345                 cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
3346                 break;
3347         case RELEASE:
3348         case RELEASE_10:
3349                 /*
3350                  * The SPC-2 RELEASE does not contain a size in the SCSI CDB.
3351                  * Assume the passthrough or $FABRIC_MOD will tell us about it.
3352                 */
3353                 if (cdb[0] == RELEASE_10)
3354                         size = (cdb[7] << 8) | cdb[8];
3355                 else
3356                         size = cmd->data_length;
3357
3358                 cmd->transport_emulate_cdb =
3359                                 (su_dev->t10_pr.res_type !=
3360                                  SPC_PASSTHROUGH) ?
3361                                 core_scsi2_emulate_crh : NULL;
3362                 cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
3363                 break;
3364         case SYNCHRONIZE_CACHE:
3365         case 0x91: /* SYNCHRONIZE_CACHE_16: */
3366                 /*
3367                  * Extract LBA and range to be flushed for emulated SYNCHRONIZE_CACHE
3368                  */
3369                 if (cdb[0] == SYNCHRONIZE_CACHE) {
3370                         sectors = transport_get_sectors_10(cdb, cmd, &sector_ret);
3371                         cmd->t_task.t_task_lba = transport_lba_32(cdb);
3372                 } else {
3373                         sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
3374                         cmd->t_task.t_task_lba = transport_lba_64(cdb);
3375                 }
3376                 if (sector_ret)
3377                         goto out_unsupported_cdb;
3378
3379                 size = transport_get_size(sectors, cdb, cmd);
3380                 cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
3381
3382                 /*
3383                  * For TCM/pSCSI passthrough, skip cmd->transport_emulate_cdb()
3384                  */
3385                 if (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV)
3386                         break;
3387                 /*
3388                  * Set SCF_EMULATE_CDB_ASYNC to ensure asynchronous operation
3389                  * for SYNCHRONIZE_CACHE* Immed=1 case in __transport_execute_tasks()
3390                  */
3391                 cmd->se_cmd_flags |= SCF_EMULATE_CDB_ASYNC;
3392                 /*
3393                  * Check to ensure that LBA + Range does not exceed past end of
3394                  * device.
3395                  */
3396                 if (transport_get_sectors(cmd) < 0)
3397                         goto out_invalid_cdb_field;
3398                 break;
3399         case UNMAP:
3400                 size = get_unaligned_be16(&cdb[7]);
3401                 passthrough = (dev->transport->transport_type ==
3402                                 TRANSPORT_PLUGIN_PHBA_PDEV);
3403                 /*
3404                  * Determine if the received UNMAP used to for direct passthrough
3405                  * into Linux/SCSI with struct request via TCM/pSCSI or we are
3406                  * signaling the use of internal transport_generic_unmap() emulation
3407                  * for UNMAP -> Linux/BLOCK disbard with TCM/IBLOCK and TCM/FILEIO
3408                  * subsystem plugin backstores.
3409                  */
3410                 if (!(passthrough))
3411                         cmd->se_cmd_flags |= SCF_EMULATE_SYNC_UNMAP;
3412
3413                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3414                 break;
3415         case WRITE_SAME_16:
3416                 sectors = transport_get_sectors_16(cdb, cmd, &sector_ret);
3417                 if (sector_ret)
3418                         goto out_unsupported_cdb;
3419                 size = transport_get_size(sectors, cdb, cmd);
3420                 cmd->t_task.t_task_lba = get_unaligned_be16(&cdb[2]);
3421                 passthrough = (dev->transport->transport_type ==
3422                                 TRANSPORT_PLUGIN_PHBA_PDEV);
3423                 /*
3424                  * Determine if the received WRITE_SAME_16 is used to for direct
3425                  * passthrough into Linux/SCSI with struct request via TCM/pSCSI
3426                  * or we are signaling the use of internal WRITE_SAME + UNMAP=1
3427                  * emulation for -> Linux/BLOCK disbard with TCM/IBLOCK and
3428                  * TCM/FILEIO subsystem plugin backstores.
3429                  */
3430                 if (!(passthrough)) {
3431                         if ((cdb[1] & 0x04) || (cdb[1] & 0x02)) {
3432                                 printk(KERN_ERR "WRITE_SAME PBDATA and LBDATA"
3433                                         " bits not supported for Block Discard"
3434                                         " Emulation\n");
3435                                 goto out_invalid_cdb_field;
3436                         }
3437                         /*
3438                          * Currently for the emulated case we only accept
3439                          * tpws with the UNMAP=1 bit set.
3440                          */
3441                         if (!(cdb[1] & 0x08)) {
3442                                 printk(KERN_ERR "WRITE_SAME w/o UNMAP bit not "
3443                                         " supported for Block Discard Emulation\n");
3444                                 goto out_invalid_cdb_field;
3445                         }
3446                 }
3447                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_SG_IO_CDB;
3448                 break;
3449         case ALLOW_MEDIUM_REMOVAL:
3450         case GPCMD_CLOSE_TRACK:
3451         case ERASE:
3452         case INITIALIZE_ELEMENT_STATUS:
3453         case GPCMD_LOAD_UNLOAD:
3454         case REZERO_UNIT:
3455         case SEEK_10:
3456         case GPCMD_SET_SPEED:
3457         case SPACE:
3458         case START_STOP:
3459         case TEST_UNIT_READY:
3460         case VERIFY:
3461         case WRITE_FILEMARKS:
3462         case MOVE_MEDIUM:
3463                 cmd->se_cmd_flags |= SCF_SCSI_NON_DATA_CDB;
3464                 break;
3465         case REPORT_LUNS:
3466                 cmd->transport_emulate_cdb =
3467                                 transport_core_report_lun_response;
3468                 size = (cdb[6] << 24) | (cdb[7] << 16) | (cdb[8] << 8) | cdb[9];
3469                 /*
3470                  * Do implict HEAD_OF_QUEUE processing for REPORT_LUNS
3471                  * See spc4r17 section 5.3
3472                  */
3473                 if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
3474                         cmd->sam_task_attr = MSG_HEAD_TAG;
3475                 cmd->se_cmd_flags |= SCF_SCSI_CONTROL_NONSG_IO_CDB;
3476                 break;
3477         default:
3478                 printk(KERN_WARNING "TARGET_CORE[%s]: Unsupported SCSI Opcode"
3479                         " 0x%02x, sending CHECK_CONDITION.\n",
3480                         cmd->se_tfo->get_fabric_name(), cdb[0]);
3481                 cmd->transport_wait_for_tasks = &transport_nop_wait_for_tasks;
3482                 goto out_unsupported_cdb;
3483         }
3484
3485         if (size != cmd->data_length) {
3486                 printk(KERN_WARNING "TARGET_CORE[%s]: Expected Transfer Length:"
3487                         " %u does not match SCSI CDB Length: %u for SAM Opcode:"
3488                         " 0x%02x\n", cmd->se_tfo->get_fabric_name(),
3489                                 cmd->data_length, size, cdb[0]);
3490
3491                 cmd->cmd_spdtl = size;
3492
3493                 if (cmd->data_direction == DMA_TO_DEVICE) {
3494                         printk(KERN_ERR "Rejecting underflow/overflow"
3495                                         " WRITE data\n");
3496                         goto out_invalid_cdb_field;
3497                 }
3498                 /*
3499                  * Reject READ_* or WRITE_* with overflow/underflow for
3500                  * type SCF_SCSI_DATA_SG_IO_CDB.
3501                  */
3502                 if (!(ret) && (dev->se_sub_dev->se_dev_attrib.block_size != 512))  {
3503                         printk(KERN_ERR "Failing OVERFLOW/UNDERFLOW for LBA op"
3504                                 " CDB on non 512-byte sector setup subsystem"
3505                                 " plugin: %s\n", dev->transport->name);
3506                         /* Returns CHECK_CONDITION + INVALID_CDB_FIELD */
3507                         goto out_invalid_cdb_field;
3508                 }
3509
3510                 if (size > cmd->data_length) {
3511                         cmd->se_cmd_flags |= SCF_OVERFLOW_BIT;
3512                         cmd->residual_count = (size - cmd->data_length);
3513                 } else {
3514                         cmd->se_cmd_flags |= SCF_UNDERFLOW_BIT;
3515                         cmd->residual_count = (cmd->data_length - size);
3516                 }
3517                 cmd->data_length = size;
3518         }
3519
3520         transport_set_supported_SAM_opcode(cmd);
3521         return ret;
3522
3523 out_unsupported_cdb:
3524         cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
3525         cmd->scsi_sense_reason = TCM_UNSUPPORTED_SCSI_OPCODE;
3526         return -EINVAL;
3527 out_invalid_cdb_field:
3528         cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
3529         cmd->scsi_sense_reason = TCM_INVALID_CDB_FIELD;
3530         return -EINVAL;
3531 }
3532
3533 static inline void transport_release_tasks(struct se_cmd *);
3534
3535 /*
3536  * This function will copy a contiguous *src buffer into a destination
3537  * struct scatterlist array.
3538  */
3539 static void transport_memcpy_write_contig(
3540         struct se_cmd *cmd,
3541         struct scatterlist *sg_d,
3542         unsigned char *src)
3543 {
3544         u32 i = 0, length = 0, total_length = cmd->data_length;
3545         void *dst;
3546
3547         while (total_length) {
3548                 length = sg_d[i].length;
3549
3550                 if (length > total_length)
3551                         length = total_length;
3552
3553                 dst = sg_virt(&sg_d[i]);
3554
3555                 memcpy(dst, src, length);
3556
3557                 if (!(total_length -= length))
3558                         return;
3559
3560                 src += length;
3561                 i++;
3562         }
3563 }
3564
3565 /*
3566  * This function will copy a struct scatterlist array *sg_s into a destination
3567  * contiguous *dst buffer.
3568  */
3569 static void transport_memcpy_read_contig(
3570         struct se_cmd *cmd,
3571         unsigned char *dst,
3572         struct scatterlist *sg_s)
3573 {
3574         u32 i = 0, length = 0, total_length = cmd->data_length;
3575         void *src;
3576
3577         while (total_length) {
3578                 length = sg_s[i].length;
3579
3580                 if (length > total_length)
3581                         length = total_length;
3582
3583                 src = sg_virt(&sg_s[i]);
3584
3585                 memcpy(dst, src, length);
3586
3587                 if (!(total_length -= length))
3588                         return;
3589
3590                 dst += length;
3591                 i++;
3592         }
3593 }
3594
3595 static void transport_memcpy_se_mem_read_contig(
3596         struct se_cmd *cmd,
3597         unsigned char *dst,
3598         struct list_head *se_mem_list)
3599 {
3600         struct se_mem *se_mem;
3601         void *src;
3602         u32 length = 0, total_length = cmd->data_length;
3603
3604         list_for_each_entry(se_mem, se_mem_list, se_list) {
3605                 length = se_mem->se_len;
3606
3607                 if (length > total_length)
3608                         length = total_length;
3609
3610                 src = page_address(se_mem->se_page) + se_mem->se_off;
3611
3612                 memcpy(dst, src, length);
3613
3614                 if (!(total_length -= length))
3615                         return;
3616
3617                 dst += length;
3618         }
3619 }
3620
3621 /*
3622  * Called from transport_generic_complete_ok() and
3623  * transport_generic_request_failure() to determine which dormant/delayed
3624  * and ordered cmds need to have their tasks added to the execution queue.
3625  */
3626 static void transport_complete_task_attr(struct se_cmd *cmd)
3627 {
3628         struct se_device *dev = cmd->se_dev;
3629         struct se_cmd *cmd_p, *cmd_tmp;
3630         int new_active_tasks = 0;
3631
3632         if (cmd->sam_task_attr == MSG_SIMPLE_TAG) {
3633                 atomic_dec(&dev->simple_cmds);
3634                 smp_mb__after_atomic_dec();
3635                 dev->dev_cur_ordered_id++;
3636                 DEBUG_STA("Incremented dev->dev_cur_ordered_id: %u for"
3637                         " SIMPLE: %u\n", dev->dev_cur_ordered_id,
3638                         cmd->se_ordered_id);
3639         } else if (cmd->sam_task_attr == MSG_HEAD_TAG) {
3640                 atomic_dec(&dev->dev_hoq_count);
3641                 smp_mb__after_atomic_dec();
3642                 dev->dev_cur_ordered_id++;
3643                 DEBUG_STA("Incremented dev_cur_ordered_id: %u for"
3644                         " HEAD_OF_QUEUE: %u\n", dev->dev_cur_ordered_id,
3645                         cmd->se_ordered_id);
3646         } else if (cmd->sam_task_attr == MSG_ORDERED_TAG) {
3647                 spin_lock(&dev->ordered_cmd_lock);
3648                 list_del(&cmd->se_ordered_node);
3649                 atomic_dec(&dev->dev_ordered_sync);
3650                 smp_mb__after_atomic_dec();
3651                 spin_unlock(&dev->ordered_cmd_lock);
3652
3653                 dev->dev_cur_ordered_id++;
3654                 DEBUG_STA("Incremented dev_cur_ordered_id: %u for ORDERED:"
3655                         " %u\n", dev->dev_cur_ordered_id, cmd->se_ordered_id);
3656         }
3657         /*
3658          * Process all commands up to the last received
3659          * ORDERED task attribute which requires another blocking
3660          * boundary
3661          */
3662         spin_lock(&dev->delayed_cmd_lock);
3663         list_for_each_entry_safe(cmd_p, cmd_tmp,
3664                         &dev->delayed_cmd_list, se_delayed_node) {
3665
3666                 list_del(&cmd_p->se_delayed_node);
3667                 spin_unlock(&dev->delayed_cmd_lock);
3668
3669                 DEBUG_STA("Calling add_tasks() for"
3670                         " cmd_p: 0x%02x Task Attr: 0x%02x"
3671                         " Dormant -> Active, se_ordered_id: %u\n",
3672                         T_TASK(cmd_p)->t_task_cdb[0],
3673                         cmd_p->sam_task_attr, cmd_p->se_ordered_id);
3674
3675                 transport_add_tasks_from_cmd(cmd_p);
3676                 new_active_tasks++;
3677
3678                 spin_lock(&dev->delayed_cmd_lock);
3679                 if (cmd_p->sam_task_attr == MSG_ORDERED_TAG)
3680                         break;
3681         }
3682         spin_unlock(&dev->delayed_cmd_lock);
3683         /*
3684          * If new tasks have become active, wake up the transport thread
3685          * to do the processing of the Active tasks.
3686          */
3687         if (new_active_tasks != 0)
3688                 wake_up_interruptible(&dev->dev_queue_obj.thread_wq);
3689 }
3690
3691 static void transport_generic_complete_ok(struct se_cmd *cmd)
3692 {
3693         int reason = 0;
3694         /*
3695          * Check if we need to move delayed/dormant tasks from cmds on the
3696          * delayed execution list after a HEAD_OF_QUEUE or ORDERED Task
3697          * Attribute.
3698          */
3699         if (cmd->se_dev->dev_task_attr_type == SAM_TASK_ATTR_EMULATED)
3700                 transport_complete_task_attr(cmd);
3701         /*
3702          * Check if we need to retrieve a sense buffer from
3703          * the struct se_cmd in question.
3704          */
3705         if (cmd->se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) {
3706                 if (transport_get_sense_data(cmd) < 0)
3707                         reason = TCM_NON_EXISTENT_LUN;
3708
3709                 /*
3710                  * Only set when an struct se_task->task_scsi_status returned
3711                  * a non GOOD status.
3712                  */
3713                 if (cmd->scsi_status) {
3714                         transport_send_check_condition_and_sense(
3715                                         cmd, reason, 1);
3716                         transport_lun_remove_cmd(cmd);
3717                         transport_cmd_check_stop_to_fabric(cmd);
3718                         return;
3719                 }
3720         }
3721         /*
3722          * Check for a callback, used by amongst other things
3723          * XDWRITE_READ_10 emulation.
3724          */
3725         if (cmd->transport_complete_callback)
3726                 cmd->transport_complete_callback(cmd);
3727
3728         switch (cmd->data_direction) {
3729         case DMA_FROM_DEVICE:
3730                 spin_lock(&cmd->se_lun->lun_sep_lock);
3731                 if (cmd->se_lun->lun_sep) {
3732                         cmd->se_lun->lun_sep->sep_stats.tx_data_octets +=
3733                                         cmd->data_length;
3734                 }
3735                 spin_unlock(&cmd->se_lun->lun_sep_lock);
3736                 /*
3737                  * If enabled by TCM fabirc module pre-registered SGL
3738                  * memory, perform the memcpy() from the TCM internal
3739                  * contigious buffer back to the original SGL.
3740                  */
3741                 if (cmd->se_cmd_flags & SCF_PASSTHROUGH_CONTIG_TO_SG)
3742                         transport_memcpy_write_contig(cmd,
3743                                  cmd->t_task.t_task_pt_sgl,
3744                                  cmd->t_task.t_task_buf);
3745
3746                 cmd->se_tfo->queue_data_in(cmd);
3747                 break;
3748         case DMA_TO_DEVICE:
3749                 spin_lock(&cmd->se_lun->lun_sep_lock);
3750                 if (cmd->se_lun->lun_sep) {
3751                         cmd->se_lun->lun_sep->sep_stats.rx_data_octets +=
3752                                 cmd->data_length;
3753                 }
3754                 spin_unlock(&cmd->se_lun->lun_sep_lock);
3755                 /*
3756                  * Check if we need to send READ payload for BIDI-COMMAND
3757                  */
3758                 if (!list_empty(&cmd->t_task.t_mem_bidi_list)) {
3759                         spin_lock(&cmd->se_lun->lun_sep_lock);
3760                         if (cmd->se_lun->lun_sep) {
3761                                 cmd->se_lun->lun_sep->sep_stats.tx_data_octets +=
3762                                         cmd->data_length;
3763                         }
3764                         spin_unlock(&cmd->se_lun->lun_sep_lock);
3765                         cmd->se_tfo->queue_data_in(cmd);
3766                         break;
3767                 }
3768                 /* Fall through for DMA_TO_DEVICE */
3769         case DMA_NONE:
3770                 cmd->se_tfo->queue_status(cmd);
3771                 break;
3772         default:
3773                 break;
3774         }
3775
3776         transport_lun_remove_cmd(cmd);
3777         transport_cmd_check_stop_to_fabric(cmd);
3778 }
3779
3780 static void transport_free_dev_tasks(struct se_cmd *cmd)
3781 {
3782         struct se_task *task, *task_tmp;
3783         unsigned long flags;
3784
3785         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
3786         list_for_each_entry_safe(task, task_tmp,
3787                                 &cmd->t_task.t_task_list, t_list) {
3788                 if (atomic_read(&task->task_active))
3789                         continue;
3790
3791                 kfree(task->task_sg_bidi);
3792                 kfree(task->task_sg);
3793
3794                 list_del(&task->t_list);
3795
3796                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
3797                 if (task->se_dev)
3798                         task->se_dev->transport->free_task(task);
3799                 else
3800                         printk(KERN_ERR "task[%u] - task->se_dev is NULL\n",
3801                                 task->task_no);
3802                 spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
3803         }
3804         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
3805 }
3806
3807 static inline void transport_free_pages(struct se_cmd *cmd)
3808 {
3809         struct se_mem *se_mem, *se_mem_tmp;
3810         int free_page = 1;
3811
3812         if (cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC)
3813                 free_page = 0;
3814         if (cmd->se_dev->transport->do_se_mem_map)
3815                 free_page = 0;
3816
3817         if (cmd->t_task.t_task_buf) {
3818                 kfree(cmd->t_task.t_task_buf);
3819                 cmd->t_task.t_task_buf = NULL;
3820                 return;
3821         }
3822
3823         /*
3824          * Caller will handle releasing of struct se_mem.
3825          */
3826         if (cmd->se_cmd_flags & SCF_CMD_PASSTHROUGH_NOALLOC)
3827                 return;
3828
3829         list_for_each_entry_safe(se_mem, se_mem_tmp,
3830                         &cmd->t_task.t_mem_list, se_list) {
3831                 /*
3832                  * We only release call __free_page(struct se_mem->se_page) when
3833                  * SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC is NOT in use,
3834                  */
3835                 if (free_page)
3836                         __free_page(se_mem->se_page);
3837
3838                 list_del(&se_mem->se_list);
3839                 kmem_cache_free(se_mem_cache, se_mem);
3840         }
3841         cmd->t_task.t_tasks_se_num = 0;
3842
3843         list_for_each_entry_safe(se_mem, se_mem_tmp,
3844                                  &cmd->t_task.t_mem_bidi_list, se_list) {
3845                 /*
3846                  * We only release call __free_page(struct se_mem->se_page) when
3847                  * SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC is NOT in use,
3848                  */
3849                 if (free_page)
3850                         __free_page(se_mem->se_page);
3851
3852                 list_del(&se_mem->se_list);
3853                 kmem_cache_free(se_mem_cache, se_mem);
3854         }
3855         cmd->t_task.t_tasks_se_bidi_num = 0;
3856 }
3857
3858 static inline void transport_release_tasks(struct se_cmd *cmd)
3859 {
3860         transport_free_dev_tasks(cmd);
3861 }
3862
3863 static inline int transport_dec_and_check(struct se_cmd *cmd)
3864 {
3865         unsigned long flags;
3866
3867         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
3868         if (atomic_read(&cmd->t_task.t_fe_count)) {
3869                 if (!(atomic_dec_and_test(&cmd->t_task.t_fe_count))) {
3870                         spin_unlock_irqrestore(&cmd->t_task.t_state_lock,
3871                                         flags);
3872                         return 1;
3873                 }
3874         }
3875
3876         if (atomic_read(&cmd->t_task.t_se_count)) {
3877                 if (!(atomic_dec_and_test(&cmd->t_task.t_se_count))) {
3878                         spin_unlock_irqrestore(&cmd->t_task.t_state_lock,
3879                                         flags);
3880                         return 1;
3881                 }
3882         }
3883         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
3884
3885         return 0;
3886 }
3887
3888 static void transport_release_fe_cmd(struct se_cmd *cmd)
3889 {
3890         unsigned long flags;
3891
3892         if (transport_dec_and_check(cmd))
3893                 return;
3894
3895         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
3896         if (!(atomic_read(&cmd->t_task.transport_dev_active))) {
3897                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
3898                 goto free_pages;
3899         }
3900         atomic_set(&cmd->t_task.transport_dev_active, 0);
3901         transport_all_task_dev_remove_state(cmd);
3902         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
3903
3904         transport_release_tasks(cmd);
3905 free_pages:
3906         transport_free_pages(cmd);
3907         transport_free_se_cmd(cmd);
3908         cmd->se_tfo->release_cmd_direct(cmd);
3909 }
3910
3911 static int transport_generic_remove(
3912         struct se_cmd *cmd,
3913         int release_to_pool,
3914         int session_reinstatement)
3915 {
3916         unsigned long flags;
3917
3918         if (transport_dec_and_check(cmd)) {
3919                 if (session_reinstatement) {
3920                         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
3921                         transport_all_task_dev_remove_state(cmd);
3922                         spin_unlock_irqrestore(&cmd->t_task.t_state_lock,
3923                                         flags);
3924                 }
3925                 return 1;
3926         }
3927
3928         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
3929         if (!(atomic_read(&cmd->t_task.transport_dev_active))) {
3930                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
3931                 goto free_pages;
3932         }
3933         atomic_set(&cmd->t_task.transport_dev_active, 0);
3934         transport_all_task_dev_remove_state(cmd);
3935         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
3936
3937         transport_release_tasks(cmd);
3938
3939 free_pages:
3940         transport_free_pages(cmd);
3941
3942         if (release_to_pool) {
3943                 transport_release_cmd_to_pool(cmd);
3944         } else {
3945                 transport_free_se_cmd(cmd);
3946                 cmd->se_tfo->release_cmd_direct(cmd);
3947         }
3948
3949         return 0;
3950 }
3951
3952 /*
3953  * transport_generic_map_mem_to_cmd - Perform SGL -> struct se_mem map
3954  * @cmd:  Associated se_cmd descriptor
3955  * @mem:  SGL style memory for TCM WRITE / READ
3956  * @sg_mem_num: Number of SGL elements
3957  * @mem_bidi_in: SGL style memory for TCM BIDI READ
3958  * @sg_mem_bidi_num: Number of BIDI READ SGL elements
3959  *
3960  * Return: nonzero return cmd was rejected for -ENOMEM or inproper usage
3961  * of parameters.
3962  */
3963 int transport_generic_map_mem_to_cmd(
3964         struct se_cmd *cmd,
3965         struct scatterlist *sgl,
3966         u32 sgl_count,
3967         struct scatterlist *sgl_bidi,
3968         u32 sgl_bidi_count)
3969 {
3970         u32 mapped_sg_count = 0;
3971         int ret;
3972
3973         if (!sgl || !sgl_count)
3974                 return 0;
3975
3976         /*
3977          * Convert sgls (sgl, sgl_bidi) to list of se_mems
3978          */
3979         if ((cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB) ||
3980             (cmd->se_cmd_flags & SCF_SCSI_CONTROL_SG_IO_CDB)) {
3981                 /*
3982                  * For CDB using TCM struct se_mem linked list scatterlist memory
3983                  * processed into a TCM struct se_subsystem_dev, we do the mapping
3984                  * from the passed physical memory to struct se_mem->se_page here.
3985                  */
3986                 ret = transport_map_sg_to_mem(cmd,
3987                         &cmd->t_task.t_mem_list, sgl, &mapped_sg_count);
3988                 if (ret < 0)
3989                         return -ENOMEM;
3990
3991                 cmd->t_task.t_tasks_se_num = mapped_sg_count;
3992                 /*
3993                  * Setup BIDI READ list of struct se_mem elements
3994                  */
3995                 if (sgl_bidi && sgl_bidi_count) {
3996                         mapped_sg_count = 0;
3997                         ret = transport_map_sg_to_mem(cmd,
3998                                 &cmd->t_task.t_mem_bidi_list, sgl_bidi,
3999                                 &mapped_sg_count);
4000                         if (ret < 0)
4001                                 return -ENOMEM;
4002
4003                         cmd->t_task.t_tasks_se_bidi_num = mapped_sg_count;
4004                 }
4005                 cmd->se_cmd_flags |= SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC;
4006
4007         } else if (cmd->se_cmd_flags & SCF_SCSI_CONTROL_NONSG_IO_CDB) {
4008                 if (sgl_bidi || sgl_bidi_count) {
4009                         printk(KERN_ERR "BIDI-Commands not supported using "
4010                                 "SCF_SCSI_CONTROL_NONSG_IO_CDB\n");
4011                         return -ENOSYS;
4012                 }
4013                 /*
4014                  * For incoming CDBs using a contiguous buffer internall with TCM,
4015                  * save the passed struct scatterlist memory.  After TCM storage object
4016                  * processing has completed for this struct se_cmd, TCM core will call
4017                  * transport_memcpy_[write,read]_contig() as necessary from
4018                  * transport_generic_complete_ok() and transport_write_pending() in order
4019                  * to copy the TCM buffer to/from the original passed *mem in SGL ->
4020                  * struct scatterlist format.
4021                  */
4022                 cmd->se_cmd_flags |= SCF_PASSTHROUGH_CONTIG_TO_SG;
4023                 cmd->t_task.t_task_pt_sgl = sgl;
4024                 /* don't need sgl count? We assume it contains cmd->data_length data */
4025         }
4026
4027         return 0;
4028 }
4029 EXPORT_SYMBOL(transport_generic_map_mem_to_cmd);
4030
4031
4032 static inline long long transport_dev_end_lba(struct se_device *dev)
4033 {
4034         return dev->transport->get_blocks(dev) + 1;
4035 }
4036
4037 static int transport_get_sectors(struct se_cmd *cmd)
4038 {
4039         struct se_device *dev = cmd->se_dev;
4040
4041         cmd->t_task.t_tasks_sectors =
4042                 (cmd->data_length / dev->se_sub_dev->se_dev_attrib.block_size);
4043         if (!(cmd->t_task.t_tasks_sectors))
4044                 cmd->t_task.t_tasks_sectors = 1;
4045
4046         if (dev->transport->get_device_type(dev) != TYPE_DISK)
4047                 return 0;
4048
4049         if ((cmd->t_task.t_task_lba + cmd->t_task.t_tasks_sectors) >
4050              transport_dev_end_lba(dev)) {
4051                 printk(KERN_ERR "LBA: %llu Sectors: %u exceeds"
4052                         " transport_dev_end_lba(): %llu\n",
4053                         cmd->t_task.t_task_lba, cmd->t_task.t_tasks_sectors,
4054                         transport_dev_end_lba(dev));
4055                 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
4056                 cmd->scsi_sense_reason = TCM_SECTOR_COUNT_TOO_MANY;
4057                 return PYX_TRANSPORT_REQ_TOO_MANY_SECTORS;
4058         }
4059
4060         return 0;
4061 }
4062
4063 static int transport_new_cmd_obj(struct se_cmd *cmd)
4064 {
4065         struct se_device *dev = cmd->se_dev;
4066         u32 task_cdbs = 0, rc;
4067
4068         if (!(cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB)) {
4069                 task_cdbs++;
4070                 cmd->t_task.t_task_cdbs++;
4071         } else {
4072                 int set_counts = 1;
4073
4074                 /*
4075                  * Setup any BIDI READ tasks and memory from
4076                  * cmd->t_task.t_mem_bidi_list so the READ struct se_tasks
4077                  * are queued first for the non pSCSI passthrough case.
4078                  */
4079                 if (!list_empty(&cmd->t_task.t_mem_bidi_list) &&
4080                     (dev->transport->transport_type != TRANSPORT_PLUGIN_PHBA_PDEV)) {
4081                         rc = transport_generic_get_cdb_count(cmd,
4082                                 cmd->t_task.t_task_lba,
4083                                 cmd->t_task.t_tasks_sectors,
4084                                 DMA_FROM_DEVICE, &cmd->t_task.t_mem_bidi_list,
4085                                 set_counts);
4086                         if (!(rc)) {
4087                                 cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
4088                                 cmd->scsi_sense_reason =
4089                                         TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
4090                                 return PYX_TRANSPORT_LU_COMM_FAILURE;
4091                         }
4092                         set_counts = 0;
4093                 }
4094                 /*
4095                  * Setup the tasks and memory from cmd->t_task.t_mem_list
4096                  * Note for BIDI transfers this will contain the WRITE payload
4097                  */
4098                 task_cdbs = transport_generic_get_cdb_count(cmd,
4099                                 cmd->t_task.t_task_lba,
4100                                 cmd->t_task.t_tasks_sectors,
4101                                 cmd->data_direction, &cmd->t_task.t_mem_list,
4102                                 set_counts);
4103                 if (!(task_cdbs)) {
4104                         cmd->se_cmd_flags |= SCF_SCSI_CDB_EXCEPTION;
4105                         cmd->scsi_sense_reason =
4106                                         TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
4107                         return PYX_TRANSPORT_LU_COMM_FAILURE;
4108                 }
4109                 cmd->t_task.t_task_cdbs += task_cdbs;
4110
4111 #if 0
4112                 printk(KERN_INFO "data_length: %u, LBA: %llu t_tasks_sectors:"
4113                         " %u, t_task_cdbs: %u\n", obj_ptr, cmd->data_length,
4114                         cmd->t_task.t_task_lba, cmd->t_task.t_tasks_sectors,
4115                         cmd->t_task.t_task_cdbs);
4116 #endif
4117         }
4118
4119         atomic_set(&cmd->t_task.t_task_cdbs_left, task_cdbs);
4120         atomic_set(&cmd->t_task.t_task_cdbs_ex_left, task_cdbs);
4121         atomic_set(&cmd->t_task.t_task_cdbs_timeout_left, task_cdbs);
4122         return 0;
4123 }
4124
4125 static int
4126 transport_generic_get_mem(struct se_cmd *cmd, u32 length, u32 dma_size)
4127 {
4128         unsigned char *buf;
4129         struct se_mem *se_mem;
4130
4131         /*
4132          * If the device uses memory mapping this is enough.
4133          */
4134         if (cmd->se_dev->transport->do_se_mem_map)
4135                 return 0;
4136
4137         while (length) {
4138                 se_mem = kmem_cache_zalloc(se_mem_cache, GFP_KERNEL);
4139                 if (!(se_mem)) {
4140                         printk(KERN_ERR "Unable to allocate struct se_mem\n");
4141                         goto out;
4142                 }
4143
4144 /* #warning FIXME Allocate contigous pages for struct se_mem elements */
4145                 se_mem->se_page = alloc_pages(GFP_KERNEL, 0);
4146                 if (!(se_mem->se_page)) {
4147                         printk(KERN_ERR "alloc_pages() failed\n");
4148                         goto out;
4149                 }
4150
4151                 buf = kmap_atomic(se_mem->se_page, KM_IRQ0);
4152                 if (!(buf)) {
4153                         printk(KERN_ERR "kmap_atomic() failed\n");
4154                         goto out;
4155                 }
4156                 INIT_LIST_HEAD(&se_mem->se_list);
4157                 se_mem->se_len = (length > dma_size) ? dma_size : length;
4158                 memset(buf, 0, se_mem->se_len);
4159                 kunmap_atomic(buf, KM_IRQ0);
4160
4161                 list_add_tail(&se_mem->se_list, &cmd->t_task.t_mem_list);
4162                 cmd->t_task.t_tasks_se_num++;
4163
4164                 DEBUG_MEM("Allocated struct se_mem page(%p) Length(%u)"
4165                         " Offset(%u)\n", se_mem->se_page, se_mem->se_len,
4166                         se_mem->se_off);
4167
4168                 length -= se_mem->se_len;
4169         }
4170
4171         DEBUG_MEM("Allocated total struct se_mem elements(%u)\n",
4172                         cmd->t_task.t_tasks_se_num);
4173
4174         return 0;
4175 out:
4176         if (se_mem)
4177                 __free_pages(se_mem->se_page, 0);
4178         kmem_cache_free(se_mem_cache, se_mem);
4179         return -ENOMEM;
4180 }
4181
4182 int transport_init_task_sg(
4183         struct se_task *task,
4184         struct se_mem *in_se_mem,
4185         u32 task_offset)
4186 {
4187         struct se_cmd *se_cmd = task->task_se_cmd;
4188         struct se_device *se_dev = se_cmd->se_dev;
4189         struct se_mem *se_mem = in_se_mem;
4190         struct target_core_fabric_ops *tfo = se_cmd->se_tfo;
4191         u32 sg_length, task_size = task->task_size, task_sg_num_padded;
4192
4193         while (task_size != 0) {
4194                 DEBUG_SC("se_mem->se_page(%p) se_mem->se_len(%u)"
4195                         " se_mem->se_off(%u) task_offset(%u)\n",
4196                         se_mem->se_page, se_mem->se_len,
4197                         se_mem->se_off, task_offset);
4198
4199                 if (task_offset == 0) {
4200                         if (task_size >= se_mem->se_len) {
4201                                 sg_length = se_mem->se_len;
4202
4203                                 if (!(list_is_last(&se_mem->se_list,
4204                                                 &se_cmd->t_task.t_mem_list)))
4205                                         se_mem = list_entry(se_mem->se_list.next,
4206                                                         struct se_mem, se_list);
4207                         } else {
4208                                 sg_length = task_size;
4209                                 task_size -= sg_length;
4210                                 goto next;
4211                         }
4212
4213                         DEBUG_SC("sg_length(%u) task_size(%u)\n",
4214                                         sg_length, task_size);
4215                 } else {
4216                         if ((se_mem->se_len - task_offset) > task_size) {
4217                                 sg_length = task_size;
4218                                 task_size -= sg_length;
4219                                 goto next;
4220                          } else {
4221                                 sg_length = (se_mem->se_len - task_offset);
4222
4223                                 if (!(list_is_last(&se_mem->se_list,
4224                                                 &se_cmd->t_task.t_mem_list)))
4225                                         se_mem = list_entry(se_mem->se_list.next,
4226                                                         struct se_mem, se_list);
4227                         }
4228
4229                         DEBUG_SC("sg_length(%u) task_size(%u)\n",
4230                                         sg_length, task_size);
4231
4232                         task_offset = 0;
4233                 }
4234                 task_size -= sg_length;
4235 next:
4236                 DEBUG_SC("task[%u] - Reducing task_size to(%u)\n",
4237                         task->task_no, task_size);
4238
4239                 task->task_sg_num++;
4240         }
4241         /*
4242          * Check if the fabric module driver is requesting that all
4243          * struct se_task->task_sg[] be chained together..  If so,
4244          * then allocate an extra padding SG entry for linking and
4245          * marking the end of the chained SGL.
4246          */
4247         if (tfo->task_sg_chaining) {
4248                 task_sg_num_padded = (task->task_sg_num + 1);
4249                 task->task_padded_sg = 1;
4250         } else
4251                 task_sg_num_padded = task->task_sg_num;
4252
4253         task->task_sg = kzalloc(task_sg_num_padded *
4254                         sizeof(struct scatterlist), GFP_KERNEL);
4255         if (!(task->task_sg)) {
4256                 printk(KERN_ERR "Unable to allocate memory for"
4257                                 " task->task_sg\n");
4258                 return -ENOMEM;
4259         }
4260         sg_init_table(&task->task_sg[0], task_sg_num_padded);
4261         /*
4262          * Setup task->task_sg_bidi for SCSI READ payload for
4263          * TCM/pSCSI passthrough if present for BIDI-COMMAND
4264          */
4265         if (!list_empty(&se_cmd->t_task.t_mem_bidi_list) &&
4266             (se_dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV)) {
4267                 task->task_sg_bidi = kzalloc(task_sg_num_padded *
4268                                 sizeof(struct scatterlist), GFP_KERNEL);
4269                 if (!(task->task_sg_bidi)) {
4270                         kfree(task->task_sg);
4271                         task->task_sg = NULL;
4272                         printk(KERN_ERR "Unable to allocate memory for"
4273                                 " task->task_sg_bidi\n");
4274                         return -ENOMEM;
4275                 }
4276                 sg_init_table(&task->task_sg_bidi[0], task_sg_num_padded);
4277         }
4278         /*
4279          * For the chaining case, setup the proper end of SGL for the
4280          * initial submission struct task into struct se_subsystem_api.
4281          * This will be cleared later by transport_do_task_sg_chain()
4282          */
4283         if (task->task_padded_sg) {
4284                 sg_mark_end(&task->task_sg[task->task_sg_num - 1]);
4285                 /*
4286                  * Added the 'if' check before marking end of bi-directional
4287                  * scatterlist (which gets created only in case of request
4288                  * (RD + WR).
4289                  */
4290                 if (task->task_sg_bidi)
4291                         sg_mark_end(&task->task_sg_bidi[task->task_sg_num - 1]);
4292         }
4293
4294         DEBUG_SC("Successfully allocated task->task_sg_num(%u),"
4295                 " task_sg_num_padded(%u)\n", task->task_sg_num,
4296                 task_sg_num_padded);
4297
4298         return task->task_sg_num;
4299 }
4300
4301 static inline int transport_set_tasks_sectors_disk(
4302         struct se_task *task,
4303         struct se_device *dev,
4304         unsigned long long lba,
4305         u32 sectors,
4306         int *max_sectors_set)
4307 {
4308         if ((lba + sectors) > transport_dev_end_lba(dev)) {
4309                 task->task_sectors = ((transport_dev_end_lba(dev) - lba) + 1);
4310
4311                 if (task->task_sectors > dev->se_sub_dev->se_dev_attrib.max_sectors) {
4312                         task->task_sectors = dev->se_sub_dev->se_dev_attrib.max_sectors;
4313                         *max_sectors_set = 1;
4314                 }
4315         } else {
4316                 if (sectors > dev->se_sub_dev->se_dev_attrib.max_sectors) {
4317                         task->task_sectors = dev->se_sub_dev->se_dev_attrib.max_sectors;
4318                         *max_sectors_set = 1;
4319                 } else
4320                         task->task_sectors = sectors;
4321         }
4322
4323         return 0;
4324 }
4325
4326 static inline int transport_set_tasks_sectors_non_disk(
4327         struct se_task *task,
4328         struct se_device *dev,
4329         unsigned long long lba,
4330         u32 sectors,
4331         int *max_sectors_set)
4332 {
4333         if (sectors > dev->se_sub_dev->se_dev_attrib.max_sectors) {
4334                 task->task_sectors = dev->se_sub_dev->se_dev_attrib.max_sectors;
4335                 *max_sectors_set = 1;
4336         } else
4337                 task->task_sectors = sectors;
4338
4339         return 0;
4340 }
4341
4342 static inline int transport_set_tasks_sectors(
4343         struct se_task *task,
4344         struct se_device *dev,
4345         unsigned long long lba,
4346         u32 sectors,
4347         int *max_sectors_set)
4348 {
4349         return (dev->transport->get_device_type(dev) == TYPE_DISK) ?
4350                 transport_set_tasks_sectors_disk(task, dev, lba, sectors,
4351                                 max_sectors_set) :
4352                 transport_set_tasks_sectors_non_disk(task, dev, lba, sectors,
4353                                 max_sectors_set);
4354 }
4355
4356 /*
4357  * Convert a sgl into a linked list of se_mems.
4358  */
4359 static int transport_map_sg_to_mem(
4360         struct se_cmd *cmd,
4361         struct list_head *se_mem_list,
4362         struct scatterlist *sg,
4363         u32 *sg_count)
4364 {
4365         struct se_mem *se_mem;
4366         u32 cmd_size = cmd->data_length;
4367
4368         WARN_ON(!sg);
4369
4370         while (cmd_size) {
4371                 /*
4372                  * NOTE: it is safe to return -ENOMEM at any time in creating this
4373                  * list because transport_free_pages() will eventually be called, and is
4374                  * smart enough to deallocate all list items for sg and sg_bidi lists.
4375                  */
4376                 se_mem = kmem_cache_zalloc(se_mem_cache, GFP_KERNEL);
4377                 if (!(se_mem)) {
4378                         printk(KERN_ERR "Unable to allocate struct se_mem\n");
4379                         return -ENOMEM;
4380                 }
4381                 INIT_LIST_HEAD(&se_mem->se_list);
4382                 DEBUG_MEM("sg_to_mem: Starting loop with cmd_size: %u"
4383                         " sg_page: %p offset: %d length: %d\n", cmd_size,
4384                         sg_page(sg), sg->offset, sg->length);
4385
4386                 se_mem->se_page = sg_page(sg);
4387                 se_mem->se_off = sg->offset;
4388
4389                 if (cmd_size > sg->length) {
4390                         se_mem->se_len = sg->length;
4391                         sg = sg_next(sg);
4392                 } else
4393                         se_mem->se_len = cmd_size;
4394
4395                 cmd_size -= se_mem->se_len;
4396                 (*sg_count)++;
4397
4398                 DEBUG_MEM("sg_to_mem: sg_count: %u cmd_size: %u\n",
4399                                 sg_count, cmd_size);
4400                 DEBUG_MEM("sg_to_mem: Final se_page: %p se_off: %d se_len: %d\n",
4401                                 se_mem->se_page, se_mem->se_off, se_mem->se_len);
4402
4403                 list_add_tail(&se_mem->se_list, se_mem_list);
4404         }
4405
4406         DEBUG_MEM("task[0] - Mapped(%u) struct scatterlist segments\n", sg_count);
4407
4408         return 0;
4409 }
4410
4411 /*      transport_map_mem_to_sg():
4412  *
4413  *
4414  */
4415 int transport_map_mem_to_sg(
4416         struct se_task *task,
4417         struct list_head *se_mem_list,
4418         void *in_mem,
4419         struct se_mem *in_se_mem,
4420         struct se_mem **out_se_mem,
4421         u32 *se_mem_cnt,
4422         u32 *task_offset)
4423 {
4424         struct se_cmd *se_cmd = task->task_se_cmd;
4425         struct se_mem *se_mem = in_se_mem;
4426         struct scatterlist *sg = (struct scatterlist *)in_mem;
4427         u32 task_size = task->task_size, sg_no = 0;
4428
4429         if (!sg) {
4430                 printk(KERN_ERR "Unable to locate valid struct"
4431                                 " scatterlist pointer\n");
4432                 return -EINVAL;
4433         }
4434
4435         while (task_size != 0) {
4436                 /*
4437                  * Setup the contigious array of scatterlists for
4438                  * this struct se_task.
4439                  */
4440                 sg_assign_page(sg, se_mem->se_page);
4441
4442                 if (*task_offset == 0) {
4443                         sg->offset = se_mem->se_off;
4444
4445                         if (task_size >= se_mem->se_len) {
4446                                 sg->length = se_mem->se_len;
4447
4448                                 if (!(list_is_last(&se_mem->se_list,
4449                                                 &se_cmd->t_task.t_mem_list))) {
4450                                         se_mem = list_entry(se_mem->se_list.next,
4451                                                         struct se_mem, se_list);
4452                                         (*se_mem_cnt)++;
4453                                 }
4454                         } else {
4455                                 sg->length = task_size;
4456                                 /*
4457                                  * Determine if we need to calculate an offset
4458                                  * into the struct se_mem on the next go around..
4459                                  */
4460                                 task_size -= sg->length;
4461                                 if (!(task_size))
4462                                         *task_offset = sg->length;
4463
4464                                 goto next;
4465                         }
4466
4467                 } else {
4468                         sg->offset = (*task_offset + se_mem->se_off);
4469
4470                         if ((se_mem->se_len - *task_offset) > task_size) {
4471                                 sg->length = task_size;
4472                                 /*
4473                                  * Determine if we need to calculate an offset
4474                                  * into the struct se_mem on the next go around..
4475                                  */
4476                                 task_size -= sg->length;
4477                                 if (!(task_size))
4478                                         *task_offset += sg->length;
4479
4480                                 goto next;
4481                         } else {
4482                                 sg->length = (se_mem->se_len - *task_offset);
4483
4484                                 if (!(list_is_last(&se_mem->se_list,
4485                                                 &se_cmd->t_task.t_mem_list))) {
4486                                         se_mem = list_entry(se_mem->se_list.next,
4487                                                         struct se_mem, se_list);
4488                                         (*se_mem_cnt)++;
4489                                 }
4490                         }
4491
4492                         *task_offset = 0;
4493                 }
4494                 task_size -= sg->length;
4495 next:
4496                 DEBUG_MEM("task[%u] mem_to_sg - sg[%u](%p)(%u)(%u) - Reducing"
4497                         " task_size to(%u), task_offset: %u\n", task->task_no, sg_no,
4498                         sg_page(sg), sg->length, sg->offset, task_size, *task_offset);
4499
4500                 sg_no++;
4501                 if (!(task_size))
4502                         break;
4503
4504                 sg = sg_next(sg);
4505
4506                 if (task_size > se_cmd->data_length)
4507                         BUG();
4508         }
4509         *out_se_mem = se_mem;
4510
4511         DEBUG_MEM("task[%u] - Mapped(%u) struct se_mem segments to total(%u)"
4512                 " SGs\n", task->task_no, *se_mem_cnt, sg_no);
4513
4514         return 0;
4515 }
4516
4517 /*
4518  * This function can be used by HW target mode drivers to create a linked
4519  * scatterlist from all contiguously allocated struct se_task->task_sg[].
4520  * This is intended to be called during the completion path by TCM Core
4521  * when struct target_core_fabric_ops->check_task_sg_chaining is enabled.
4522  */
4523 void transport_do_task_sg_chain(struct se_cmd *cmd)
4524 {
4525         struct scatterlist *sg_head = NULL, *sg_link = NULL, *sg_first = NULL;
4526         struct scatterlist *sg_head_cur = NULL, *sg_link_cur = NULL;
4527         struct scatterlist *sg, *sg_end = NULL, *sg_end_cur = NULL;
4528         struct se_task *task;
4529         struct target_core_fabric_ops *tfo = cmd->se_tfo;
4530         u32 task_sg_num = 0, sg_count = 0;
4531         int i;
4532
4533         if (tfo->task_sg_chaining == 0) {
4534                 printk(KERN_ERR "task_sg_chaining is diabled for fabric module:"
4535                                 " %s\n", tfo->get_fabric_name());
4536                 dump_stack();
4537                 return;
4538         }
4539         /*
4540          * Walk the struct se_task list and setup scatterlist chains
4541          * for each contiguosly allocated struct se_task->task_sg[].
4542          */
4543         list_for_each_entry(task, &cmd->t_task.t_task_list, t_list) {
4544                 if (!(task->task_sg) || !(task->task_padded_sg))
4545                         continue;
4546
4547                 if (sg_head && sg_link) {
4548                         sg_head_cur = &task->task_sg[0];
4549                         sg_link_cur = &task->task_sg[task->task_sg_num];
4550                         /*
4551                          * Either add chain or mark end of scatterlist
4552                          */
4553                         if (!(list_is_last(&task->t_list,
4554                                         &cmd->t_task.t_task_list))) {
4555                                 /*
4556                                  * Clear existing SGL termination bit set in
4557                                  * transport_init_task_sg(), see sg_mark_end()
4558                                  */
4559                                 sg_end_cur = &task->task_sg[task->task_sg_num - 1];
4560                                 sg_end_cur->page_link &= ~0x02;
4561
4562                                 sg_chain(sg_head, task_sg_num, sg_head_cur);
4563                                 sg_count += task->task_sg_num;
4564                                 task_sg_num = (task->task_sg_num + 1);
4565                         } else {
4566                                 sg_chain(sg_head, task_sg_num, sg_head_cur);
4567                                 sg_count += task->task_sg_num;
4568                                 task_sg_num = task->task_sg_num;
4569                         }
4570
4571                         sg_head = sg_head_cur;
4572                         sg_link = sg_link_cur;
4573                         continue;
4574                 }
4575                 sg_head = sg_first = &task->task_sg[0];
4576                 sg_link = &task->task_sg[task->task_sg_num];
4577                 /*
4578                  * Check for single task..
4579                  */
4580                 if (!(list_is_last(&task->t_list, &cmd->t_task.t_task_list))) {
4581                         /*
4582                          * Clear existing SGL termination bit set in
4583                          * transport_init_task_sg(), see sg_mark_end()
4584                          */
4585                         sg_end = &task->task_sg[task->task_sg_num - 1];
4586                         sg_end->page_link &= ~0x02;
4587                         sg_count += task->task_sg_num;
4588                         task_sg_num = (task->task_sg_num + 1);
4589                 } else {
4590                         sg_count += task->task_sg_num;
4591                         task_sg_num = task->task_sg_num;
4592                 }
4593         }
4594         /*
4595          * Setup the starting pointer and total t_tasks_sg_linked_no including
4596          * padding SGs for linking and to mark the end.
4597          */
4598         cmd->t_task.t_tasks_sg_chained = sg_first;
4599         cmd->t_task.t_tasks_sg_chained_no = sg_count;
4600
4601         DEBUG_CMD_M("Setup cmd: %p cmd->t_task.t_tasks_sg_chained: %p and"
4602                 " t_tasks_sg_chained_no: %u\n", cmd, cmd->t_task.t_tasks_sg_chained,
4603                 cmd->t_task.t_tasks_sg_chained_no);
4604
4605         for_each_sg(cmd->t_task.t_tasks_sg_chained, sg,
4606                         cmd->t_task.t_tasks_sg_chained_no, i) {
4607
4608                 DEBUG_CMD_M("SG[%d]: %p page: %p length: %d offset: %d\n",
4609                         i, sg, sg_page(sg), sg->length, sg->offset);
4610                 if (sg_is_chain(sg))
4611                         DEBUG_CMD_M("SG: %p sg_is_chain=1\n", sg);
4612                 if (sg_is_last(sg))
4613                         DEBUG_CMD_M("SG: %p sg_is_last=1\n", sg);
4614         }
4615 }
4616 EXPORT_SYMBOL(transport_do_task_sg_chain);
4617
4618 static int transport_do_se_mem_map(
4619         struct se_device *dev,
4620         struct se_task *task,
4621         struct list_head *se_mem_list,
4622         void *in_mem,
4623         struct se_mem *in_se_mem,
4624         struct se_mem **out_se_mem,
4625         u32 *se_mem_cnt,
4626         u32 *task_offset_in)
4627 {
4628         u32 task_offset = *task_offset_in;
4629         int ret = 0;
4630         /*
4631          * se_subsystem_api_t->do_se_mem_map is used when internal allocation
4632          * has been done by the transport plugin.
4633          */
4634         if (dev->transport->do_se_mem_map) {
4635                 ret = dev->transport->do_se_mem_map(task, se_mem_list,
4636                                 in_mem, in_se_mem, out_se_mem, se_mem_cnt,
4637                                 task_offset_in);
4638                 if (ret == 0)
4639                         task->task_se_cmd->t_task.t_tasks_se_num += *se_mem_cnt;
4640
4641                 return ret;
4642         }
4643
4644         BUG_ON(list_empty(se_mem_list));
4645         /*
4646          * This is the normal path for all normal non BIDI and BIDI-COMMAND
4647          * WRITE payloads..  If we need to do BIDI READ passthrough for
4648          * TCM/pSCSI the first call to transport_do_se_mem_map ->
4649          * transport_init_task_sg() -> transport_map_mem_to_sg() will do the
4650          * allocation for task->task_sg_bidi, and the subsequent call to
4651          * transport_do_se_mem_map() from transport_generic_get_cdb_count()
4652          */
4653         if (!(task->task_sg_bidi)) {
4654                 /*
4655                  * Assume default that transport plugin speaks preallocated
4656                  * scatterlists.
4657                  */
4658                 ret = transport_init_task_sg(task, in_se_mem, task_offset);
4659                 if (ret <= 0)
4660                         return ret;
4661                 /*
4662                  * struct se_task->task_sg now contains the struct scatterlist array.
4663                  */
4664                 return transport_map_mem_to_sg(task, se_mem_list, task->task_sg,
4665                                         in_se_mem, out_se_mem, se_mem_cnt,
4666                                         task_offset_in);
4667         }
4668         /*
4669          * Handle the se_mem_list -> struct task->task_sg_bidi
4670          * memory map for the extra BIDI READ payload
4671          */
4672         return transport_map_mem_to_sg(task, se_mem_list, task->task_sg_bidi,
4673                                 in_se_mem, out_se_mem, se_mem_cnt,
4674                                 task_offset_in);
4675 }
4676
4677 static u32 transport_generic_get_cdb_count(
4678         struct se_cmd *cmd,
4679         unsigned long long lba,
4680         u32 sectors,
4681         enum dma_data_direction data_direction,
4682         struct list_head *mem_list,
4683         int set_counts)
4684 {
4685         unsigned char *cdb = NULL;
4686         struct se_task *task;
4687         struct se_mem *se_mem = NULL, *se_mem_lout = NULL;
4688         struct se_mem *se_mem_bidi = NULL, *se_mem_bidi_lout = NULL;
4689         struct se_device *dev = cmd->se_dev;
4690         int max_sectors_set = 0, ret;
4691         u32 task_offset_in = 0, se_mem_cnt = 0, se_mem_bidi_cnt = 0, task_cdbs = 0;
4692
4693         if (!mem_list) {
4694                 printk(KERN_ERR "mem_list is NULL in transport_generic_get"
4695                                 "_cdb_count()\n");
4696                 return 0;
4697         }
4698         /*
4699          * While using RAMDISK_DR backstores is the only case where
4700          * mem_list will ever be empty at this point.
4701          */
4702         if (!(list_empty(mem_list)))
4703                 se_mem = list_first_entry(mem_list, struct se_mem, se_list);
4704         /*
4705          * Check for extra se_mem_bidi mapping for BIDI-COMMANDs to
4706          * struct se_task->task_sg_bidi for TCM/pSCSI passthrough operation
4707          */
4708         if (!list_empty(&cmd->t_task.t_mem_bidi_list) &&
4709             (dev->transport->transport_type == TRANSPORT_PLUGIN_PHBA_PDEV))
4710                 se_mem_bidi = list_first_entry(&cmd->t_task.t_mem_bidi_list,
4711                                         struct se_mem, se_list);
4712
4713         while (sectors) {
4714                 DEBUG_VOL("ITT[0x%08x] LBA(%llu) SectorsLeft(%u) EOBJ(%llu)\n",
4715                         cmd->se_tfo->get_task_tag(cmd), lba, sectors,
4716                         transport_dev_end_lba(dev));
4717
4718                 task = transport_generic_get_task(cmd, data_direction);
4719                 if (!(task))
4720                         goto out;
4721
4722                 transport_set_tasks_sectors(task, dev, lba, sectors,
4723                                 &max_sectors_set);
4724
4725                 task->task_lba = lba;
4726                 lba += task->task_sectors;
4727                 sectors -= task->task_sectors;
4728                 task->task_size = (task->task_sectors *
4729                                    dev->se_sub_dev->se_dev_attrib.block_size);
4730
4731                 cdb = dev->transport->get_cdb(task);
4732                 if ((cdb)) {
4733                         memcpy(cdb, cmd->t_task.t_task_cdb,
4734                                 scsi_command_size(cmd->t_task.t_task_cdb));
4735                         cmd->transport_split_cdb(task->task_lba,
4736                                         &task->task_sectors, cdb);
4737                 }
4738
4739                 /*
4740                  * Perform the SE OBJ plugin and/or Transport plugin specific
4741                  * mapping for cmd->t_task.t_mem_list. And setup the
4742                  * task->task_sg and if necessary task->task_sg_bidi
4743                  */
4744                 ret = transport_do_se_mem_map(dev, task, mem_list,
4745                                 NULL, se_mem, &se_mem_lout, &se_mem_cnt,
4746                                 &task_offset_in);
4747                 if (ret < 0)
4748                         goto out;
4749
4750                 se_mem = se_mem_lout;
4751                 /*
4752                  * Setup the cmd->t_task.t_mem_bidi_list -> task->task_sg_bidi
4753                  * mapping for SCSI READ for BIDI-COMMAND passthrough with TCM/pSCSI
4754                  *
4755                  * Note that the first call to transport_do_se_mem_map() above will
4756                  * allocate struct se_task->task_sg_bidi in transport_do_se_mem_map()
4757                  * -> transport_init_task_sg(), and the second here will do the
4758                  * mapping for SCSI READ for BIDI-COMMAND passthrough with TCM/pSCSI.
4759                  */
4760                 if (task->task_sg_bidi != NULL) {
4761                         ret = transport_do_se_mem_map(dev, task,
4762                                 &cmd->t_task.t_mem_bidi_list, NULL,
4763                                 se_mem_bidi, &se_mem_bidi_lout, &se_mem_bidi_cnt,
4764                                 &task_offset_in);
4765                         if (ret < 0)
4766                                 goto out;
4767
4768                         se_mem_bidi = se_mem_bidi_lout;
4769                 }
4770                 task_cdbs++;
4771
4772                 DEBUG_VOL("Incremented task_cdbs(%u) task->task_sg_num(%u)\n",
4773                                 task_cdbs, task->task_sg_num);
4774
4775                 if (max_sectors_set) {
4776                         max_sectors_set = 0;
4777                         continue;
4778                 }
4779
4780                 if (!sectors)
4781                         break;
4782         }
4783
4784         if (set_counts) {
4785                 atomic_inc(&cmd->t_task.t_fe_count);
4786                 atomic_inc(&cmd->t_task.t_se_count);
4787         }
4788
4789         DEBUG_VOL("ITT[0x%08x] total %s cdbs(%u)\n",
4790                 cmd->se_tfo->get_task_tag(cmd), (data_direction == DMA_TO_DEVICE)
4791                 ? "DMA_TO_DEVICE" : "DMA_FROM_DEVICE", task_cdbs);
4792
4793         return task_cdbs;
4794 out:
4795         return 0;
4796 }
4797
4798 static int
4799 transport_map_control_cmd_to_task(struct se_cmd *cmd)
4800 {
4801         struct se_device *dev = cmd->se_dev;
4802         unsigned char *cdb;
4803         struct se_task *task;
4804         int ret;
4805
4806         task = transport_generic_get_task(cmd, cmd->data_direction);
4807         if (!task)
4808                 return PYX_TRANSPORT_OUT_OF_MEMORY_RESOURCES;
4809
4810         cdb = dev->transport->get_cdb(task);
4811         if (cdb)
4812                 memcpy(cdb, cmd->t_task.t_task_cdb,
4813                         scsi_command_size(cmd->t_task.t_task_cdb));
4814
4815         task->task_size = cmd->data_length;
4816         task->task_sg_num =
4817                 (cmd->se_cmd_flags & SCF_SCSI_CONTROL_SG_IO_CDB) ? 1 : 0;
4818
4819         atomic_inc(&cmd->t_task.t_fe_count);
4820         atomic_inc(&cmd->t_task.t_se_count);
4821
4822         if (cmd->se_cmd_flags & SCF_SCSI_CONTROL_SG_IO_CDB) {
4823                 struct se_mem *se_mem = NULL, *se_mem_lout = NULL;
4824                 u32 se_mem_cnt = 0, task_offset = 0;
4825
4826                 if (!list_empty(&cmd->t_task.t_mem_list))
4827                         se_mem = list_first_entry(&cmd->t_task.t_mem_list,
4828                                         struct se_mem, se_list);
4829
4830                 ret = transport_do_se_mem_map(dev, task,
4831                                 &cmd->t_task.t_mem_list, NULL, se_mem,
4832                                 &se_mem_lout, &se_mem_cnt, &task_offset);
4833                 if (ret < 0)
4834                         return PYX_TRANSPORT_OUT_OF_MEMORY_RESOURCES;
4835
4836                 if (dev->transport->map_task_SG)
4837                         return dev->transport->map_task_SG(task);
4838                 return 0;
4839         } else if (cmd->se_cmd_flags & SCF_SCSI_CONTROL_NONSG_IO_CDB) {
4840                 if (dev->transport->map_task_non_SG)
4841                         return dev->transport->map_task_non_SG(task);
4842                 return 0;
4843         } else if (cmd->se_cmd_flags & SCF_SCSI_NON_DATA_CDB) {
4844                 if (dev->transport->cdb_none)
4845                         return dev->transport->cdb_none(task);
4846                 return 0;
4847         } else {
4848                 BUG();
4849                 return PYX_TRANSPORT_OUT_OF_MEMORY_RESOURCES;
4850         }
4851 }
4852
4853 /*       transport_generic_new_cmd(): Called from transport_processing_thread()
4854  *
4855  *       Allocate storage transport resources from a set of values predefined
4856  *       by transport_generic_cmd_sequencer() from the iSCSI Target RX process.
4857  *       Any non zero return here is treated as an "out of resource' op here.
4858  */
4859         /*
4860          * Generate struct se_task(s) and/or their payloads for this CDB.
4861          */
4862 static int transport_generic_new_cmd(struct se_cmd *cmd)
4863 {
4864         struct se_portal_group *se_tpg;
4865         struct se_task *task;
4866         struct se_device *dev = cmd->se_dev;
4867         int ret = 0;
4868
4869         /*
4870          * Determine is the TCM fabric module has already allocated physical
4871          * memory, and is directly calling transport_generic_map_mem_to_cmd()
4872          * to setup beforehand the linked list of physical memory at
4873          * cmd->t_task.t_mem_list of struct se_mem->se_page
4874          */
4875         if (!(cmd->se_cmd_flags & SCF_PASSTHROUGH_SG_TO_MEM_NOALLOC)) {
4876                 ret = transport_allocate_resources(cmd);
4877                 if (ret < 0)
4878                         return ret;
4879         }
4880
4881         ret = transport_get_sectors(cmd);
4882         if (ret < 0)
4883                 return ret;
4884
4885         ret = transport_new_cmd_obj(cmd);
4886         if (ret < 0)
4887                 return ret;
4888
4889         /*
4890          * Determine if the calling TCM fabric module is talking to
4891          * Linux/NET via kernel sockets and needs to allocate a
4892          * struct iovec array to complete the struct se_cmd
4893          */
4894         se_tpg = cmd->se_lun->lun_sep->sep_tpg;
4895         if (se_tpg->se_tpg_tfo->alloc_cmd_iovecs != NULL) {
4896                 ret = se_tpg->se_tpg_tfo->alloc_cmd_iovecs(cmd);
4897                 if (ret < 0)
4898                         return PYX_TRANSPORT_OUT_OF_MEMORY_RESOURCES;
4899         }
4900
4901         if (cmd->se_cmd_flags & SCF_SCSI_DATA_SG_IO_CDB) {
4902                 list_for_each_entry(task, &cmd->t_task.t_task_list, t_list) {
4903                         if (atomic_read(&task->task_sent))
4904                                 continue;
4905                         if (!dev->transport->map_task_SG)
4906                                 continue;
4907
4908                         ret = dev->transport->map_task_SG(task);
4909                         if (ret < 0)
4910                                 return ret;
4911                 }
4912         } else {
4913                 ret = transport_map_control_cmd_to_task(cmd);
4914                 if (ret < 0)
4915                         return ret;
4916         }
4917
4918         /*
4919          * For WRITEs, let the iSCSI Target RX Thread know its buffer is ready..
4920          * This WRITE struct se_cmd (and all of its associated struct se_task's)
4921          * will be added to the struct se_device execution queue after its WRITE
4922          * data has arrived. (ie: It gets handled by the transport processing
4923          * thread a second time)
4924          */
4925         if (cmd->data_direction == DMA_TO_DEVICE) {
4926                 transport_add_tasks_to_state_queue(cmd);
4927                 return transport_generic_write_pending(cmd);
4928         }
4929         /*
4930          * Everything else but a WRITE, add the struct se_cmd's struct se_task's
4931          * to the execution queue.
4932          */
4933         transport_execute_tasks(cmd);
4934         return 0;
4935 }
4936
4937 /*      transport_generic_process_write():
4938  *
4939  *
4940  */
4941 void transport_generic_process_write(struct se_cmd *cmd)
4942 {
4943 #if 0
4944         /*
4945          * Copy SCSI Presented DTL sector(s) from received buffers allocated to
4946          * original EDTL
4947          */
4948         if (cmd->se_cmd_flags & SCF_UNDERFLOW_BIT) {
4949                 if (!cmd->t_task.t_tasks_se_num) {
4950                         unsigned char *dst, *buf =
4951                                 (unsigned char *)cmd->t_task.t_task_buf;
4952
4953                         dst = kzalloc(cmd->cmd_spdtl), GFP_KERNEL);
4954                         if (!(dst)) {
4955                                 printk(KERN_ERR "Unable to allocate memory for"
4956                                                 " WRITE underflow\n");
4957                                 transport_generic_request_failure(cmd, NULL,
4958                                         PYX_TRANSPORT_REQ_TOO_MANY_SECTORS, 1);
4959                                 return;
4960                         }
4961                         memcpy(dst, buf, cmd->cmd_spdtl);
4962
4963                         kfree(cmd->t_task.t_task_buf);
4964                         cmd->t_task.t_task_buf = dst;
4965                 } else {
4966                         struct scatterlist *sg =
4967                                 (struct scatterlist *sg)cmd->t_task.t_task_buf;
4968                         struct scatterlist *orig_sg;
4969
4970                         orig_sg = kzalloc(sizeof(struct scatterlist) *
4971                                         cmd->t_task.t_tasks_se_num,
4972                                         GFP_KERNEL))) {
4973                         if (!(orig_sg)) {
4974                                 printk(KERN_ERR "Unable to allocate memory"
4975                                                 " for WRITE underflow\n");
4976                                 transport_generic_request_failure(cmd, NULL,
4977                                         PYX_TRANSPORT_REQ_TOO_MANY_SECTORS, 1);
4978                                 return;
4979                         }
4980
4981                         memcpy(orig_sg, cmd->t_task.t_task_buf,
4982                                         sizeof(struct scatterlist) *
4983                                         cmd->t_task.t_tasks_se_num);
4984
4985                         cmd->data_length = cmd->cmd_spdtl;
4986                         /*
4987                          * FIXME, clear out original struct se_task and state
4988                          * information.
4989                          */
4990                         if (transport_generic_new_cmd(cmd) < 0) {
4991                                 transport_generic_request_failure(cmd, NULL,
4992                                         PYX_TRANSPORT_REQ_TOO_MANY_SECTORS, 1);
4993                                 kfree(orig_sg);
4994                                 return;
4995                         }
4996
4997                         transport_memcpy_write_sg(cmd, orig_sg);
4998                 }
4999         }
5000 #endif
5001         transport_execute_tasks(cmd);
5002 }
5003 EXPORT_SYMBOL(transport_generic_process_write);
5004
5005 /*      transport_generic_write_pending():
5006  *
5007  *
5008  */
5009 static int transport_generic_write_pending(struct se_cmd *cmd)
5010 {
5011         unsigned long flags;
5012         int ret;
5013
5014         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
5015         cmd->t_state = TRANSPORT_WRITE_PENDING;
5016         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
5017         /*
5018          * For the TCM control CDBs using a contiguous buffer, do the memcpy
5019          * from the passed Linux/SCSI struct scatterlist located at
5020          * se_cmd->t_task.t_task_pt_buf to the contiguous buffer at
5021          * se_cmd->t_task.t_task_buf.
5022          */
5023         if (cmd->se_cmd_flags & SCF_PASSTHROUGH_CONTIG_TO_SG)
5024                 transport_memcpy_read_contig(cmd,
5025                                 cmd->t_task.t_task_buf,
5026                                 cmd->t_task.t_task_pt_sgl);
5027         /*
5028          * Clear the se_cmd for WRITE_PENDING status in order to set
5029          * cmd->t_task.t_transport_active=0 so that transport_generic_handle_data
5030          * can be called from HW target mode interrupt code.  This is safe
5031          * to be called with transport_off=1 before the cmd->se_tfo->write_pending
5032          * because the se_cmd->se_lun pointer is not being cleared.
5033          */
5034         transport_cmd_check_stop(cmd, 1, 0);
5035
5036         /*
5037          * Call the fabric write_pending function here to let the
5038          * frontend know that WRITE buffers are ready.
5039          */
5040         ret = cmd->se_tfo->write_pending(cmd);
5041         if (ret < 0)
5042                 return ret;
5043
5044         return PYX_TRANSPORT_WRITE_PENDING;
5045 }
5046
5047 /*      transport_release_cmd_to_pool():
5048  *
5049  *
5050  */
5051 void transport_release_cmd_to_pool(struct se_cmd *cmd)
5052 {
5053         BUG_ON(!cmd->se_tfo);
5054
5055         transport_free_se_cmd(cmd);
5056         cmd->se_tfo->release_cmd_to_pool(cmd);
5057 }
5058 EXPORT_SYMBOL(transport_release_cmd_to_pool);
5059
5060 /*      transport_generic_free_cmd():
5061  *
5062  *      Called from processing frontend to release storage engine resources
5063  */
5064 void transport_generic_free_cmd(
5065         struct se_cmd *cmd,
5066         int wait_for_tasks,
5067         int release_to_pool,
5068         int session_reinstatement)
5069 {
5070         if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD))
5071                 transport_release_cmd_to_pool(cmd);
5072         else {
5073                 core_dec_lacl_count(cmd->se_sess->se_node_acl, cmd);
5074
5075                 if (cmd->se_lun) {
5076 #if 0
5077                         printk(KERN_INFO "cmd: %p ITT: 0x%08x contains"
5078                                 " cmd->se_lun\n", cmd,
5079                                 cmd->se_tfo->get_task_tag(cmd));
5080 #endif
5081                         transport_lun_remove_cmd(cmd);
5082                 }
5083
5084                 if (wait_for_tasks && cmd->transport_wait_for_tasks)
5085                         cmd->transport_wait_for_tasks(cmd, 0, 0);
5086
5087                 transport_free_dev_tasks(cmd);
5088
5089                 transport_generic_remove(cmd, release_to_pool,
5090                                 session_reinstatement);
5091         }
5092 }
5093 EXPORT_SYMBOL(transport_generic_free_cmd);
5094
5095 static void transport_nop_wait_for_tasks(
5096         struct se_cmd *cmd,
5097         int remove_cmd,
5098         int session_reinstatement)
5099 {
5100         return;
5101 }
5102
5103 /*      transport_lun_wait_for_tasks():
5104  *
5105  *      Called from ConfigFS context to stop the passed struct se_cmd to allow
5106  *      an struct se_lun to be successfully shutdown.
5107  */
5108 static int transport_lun_wait_for_tasks(struct se_cmd *cmd, struct se_lun *lun)
5109 {
5110         unsigned long flags;
5111         int ret;
5112         /*
5113          * If the frontend has already requested this struct se_cmd to
5114          * be stopped, we can safely ignore this struct se_cmd.
5115          */
5116         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
5117         if (atomic_read(&cmd->t_task.t_transport_stop)) {
5118                 atomic_set(&cmd->t_task.transport_lun_stop, 0);
5119                 DEBUG_TRANSPORT_S("ConfigFS ITT[0x%08x] - t_transport_stop =="
5120                         " TRUE, skipping\n", cmd->se_tfo->get_task_tag(cmd));
5121                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
5122                 transport_cmd_check_stop(cmd, 1, 0);
5123                 return -EPERM;
5124         }
5125         atomic_set(&cmd->t_task.transport_lun_fe_stop, 1);
5126         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
5127
5128         wake_up_interruptible(&cmd->se_dev->dev_queue_obj.thread_wq);
5129
5130         ret = transport_stop_tasks_for_cmd(cmd);
5131
5132         DEBUG_TRANSPORT_S("ConfigFS: cmd: %p t_task_cdbs: %d stop tasks ret:"
5133                         " %d\n", cmd, cmd->t_task.t_task_cdbs, ret);
5134         if (!ret) {
5135                 DEBUG_TRANSPORT_S("ConfigFS: ITT[0x%08x] - stopping cmd....\n",
5136                                 cmd->se_tfo->get_task_tag(cmd));
5137                 wait_for_completion(&cmd->t_task.transport_lun_stop_comp);
5138                 DEBUG_TRANSPORT_S("ConfigFS: ITT[0x%08x] - stopped cmd....\n",
5139                                 cmd->se_tfo->get_task_tag(cmd));
5140         }
5141         transport_remove_cmd_from_queue(cmd, &cmd->se_dev->dev_queue_obj);
5142
5143         return 0;
5144 }
5145
5146 /* #define DEBUG_CLEAR_LUN */
5147 #ifdef DEBUG_CLEAR_LUN
5148 #define DEBUG_CLEAR_L(x...) printk(KERN_INFO x)
5149 #else
5150 #define DEBUG_CLEAR_L(x...)
5151 #endif
5152
5153 static void __transport_clear_lun_from_sessions(struct se_lun *lun)
5154 {
5155         struct se_cmd *cmd = NULL;
5156         unsigned long lun_flags, cmd_flags;
5157         /*
5158          * Do exception processing and return CHECK_CONDITION status to the
5159          * Initiator Port.
5160          */
5161         spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
5162         while (!list_empty(&lun->lun_cmd_list)) {
5163                 cmd = list_first_entry(&lun->lun_cmd_list,
5164                        struct se_cmd, se_lun_node);
5165                 list_del(&cmd->se_lun_node);
5166
5167                 atomic_set(&cmd->t_task.transport_lun_active, 0);
5168                 /*
5169                  * This will notify iscsi_target_transport.c:
5170                  * transport_cmd_check_stop() that a LUN shutdown is in
5171                  * progress for the iscsi_cmd_t.
5172                  */
5173                 spin_lock(&cmd->t_task.t_state_lock);
5174                 DEBUG_CLEAR_L("SE_LUN[%d] - Setting cmd->t_task.transport"
5175                         "_lun_stop for  ITT: 0x%08x\n",
5176                         cmd->se_lun->unpacked_lun,
5177                         cmd->se_tfo->get_task_tag(cmd));
5178                 atomic_set(&cmd->t_task.transport_lun_stop, 1);
5179                 spin_unlock(&cmd->t_task.t_state_lock);
5180
5181                 spin_unlock_irqrestore(&lun->lun_cmd_lock, lun_flags);
5182
5183                 if (!(cmd->se_lun)) {
5184                         printk(KERN_ERR "ITT: 0x%08x, [i,t]_state: %u/%u\n",
5185                                 cmd->se_tfo->get_task_tag(cmd),
5186                                 cmd->se_tfo->get_cmd_state(cmd), cmd->t_state);
5187                         BUG();
5188                 }
5189                 /*
5190                  * If the Storage engine still owns the iscsi_cmd_t, determine
5191                  * and/or stop its context.
5192                  */
5193                 DEBUG_CLEAR_L("SE_LUN[%d] - ITT: 0x%08x before transport"
5194                         "_lun_wait_for_tasks()\n", cmd->se_lun->unpacked_lun,
5195                         cmd->se_tfo->get_task_tag(cmd));
5196
5197                 if (transport_lun_wait_for_tasks(cmd, cmd->se_lun) < 0) {
5198                         spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
5199                         continue;
5200                 }
5201
5202                 DEBUG_CLEAR_L("SE_LUN[%d] - ITT: 0x%08x after transport_lun"
5203                         "_wait_for_tasks(): SUCCESS\n",
5204                         cmd->se_lun->unpacked_lun,
5205                         cmd->se_tfo->get_task_tag(cmd));
5206
5207                 spin_lock_irqsave(&cmd->t_task.t_state_lock, cmd_flags);
5208                 if (!(atomic_read(&cmd->t_task.transport_dev_active))) {
5209                         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, cmd_flags);
5210                         goto check_cond;
5211                 }
5212                 atomic_set(&cmd->t_task.transport_dev_active, 0);
5213                 transport_all_task_dev_remove_state(cmd);
5214                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, cmd_flags);
5215
5216                 transport_free_dev_tasks(cmd);
5217                 /*
5218                  * The Storage engine stopped this struct se_cmd before it was
5219                  * send to the fabric frontend for delivery back to the
5220                  * Initiator Node.  Return this SCSI CDB back with an
5221                  * CHECK_CONDITION status.
5222                  */
5223 check_cond:
5224                 transport_send_check_condition_and_sense(cmd,
5225                                 TCM_NON_EXISTENT_LUN, 0);
5226                 /*
5227                  *  If the fabric frontend is waiting for this iscsi_cmd_t to
5228                  * be released, notify the waiting thread now that LU has
5229                  * finished accessing it.
5230                  */
5231                 spin_lock_irqsave(&cmd->t_task.t_state_lock, cmd_flags);
5232                 if (atomic_read(&cmd->t_task.transport_lun_fe_stop)) {
5233                         DEBUG_CLEAR_L("SE_LUN[%d] - Detected FE stop for"
5234                                 " struct se_cmd: %p ITT: 0x%08x\n",
5235                                 lun->unpacked_lun,
5236                                 cmd, cmd->se_tfo->get_task_tag(cmd));
5237
5238                         spin_unlock_irqrestore(&cmd->t_task.t_state_lock,
5239                                         cmd_flags);
5240                         transport_cmd_check_stop(cmd, 1, 0);
5241                         complete(&cmd->t_task.transport_lun_fe_stop_comp);
5242                         spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
5243                         continue;
5244                 }
5245                 DEBUG_CLEAR_L("SE_LUN[%d] - ITT: 0x%08x finished processing\n",
5246                         lun->unpacked_lun, cmd->se_tfo->get_task_tag(cmd));
5247
5248                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, cmd_flags);
5249                 spin_lock_irqsave(&lun->lun_cmd_lock, lun_flags);
5250         }
5251         spin_unlock_irqrestore(&lun->lun_cmd_lock, lun_flags);
5252 }
5253
5254 static int transport_clear_lun_thread(void *p)
5255 {
5256         struct se_lun *lun = (struct se_lun *)p;
5257
5258         __transport_clear_lun_from_sessions(lun);
5259         complete(&lun->lun_shutdown_comp);
5260
5261         return 0;
5262 }
5263
5264 int transport_clear_lun_from_sessions(struct se_lun *lun)
5265 {
5266         struct task_struct *kt;
5267
5268         kt = kthread_run(transport_clear_lun_thread, lun,
5269                         "tcm_cl_%u", lun->unpacked_lun);
5270         if (IS_ERR(kt)) {
5271                 printk(KERN_ERR "Unable to start clear_lun thread\n");
5272                 return PTR_ERR(kt);
5273         }
5274         wait_for_completion(&lun->lun_shutdown_comp);
5275
5276         return 0;
5277 }
5278
5279 /*      transport_generic_wait_for_tasks():
5280  *
5281  *      Called from frontend or passthrough context to wait for storage engine
5282  *      to pause and/or release frontend generated struct se_cmd.
5283  */
5284 static void transport_generic_wait_for_tasks(
5285         struct se_cmd *cmd,
5286         int remove_cmd,
5287         int session_reinstatement)
5288 {
5289         unsigned long flags;
5290
5291         if (!(cmd->se_cmd_flags & SCF_SE_LUN_CMD) && !(cmd->se_tmr_req))
5292                 return;
5293
5294         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
5295         /*
5296          * If we are already stopped due to an external event (ie: LUN shutdown)
5297          * sleep until the connection can have the passed struct se_cmd back.
5298          * The cmd->t_task.transport_lun_stopped_sem will be upped by
5299          * transport_clear_lun_from_sessions() once the ConfigFS context caller
5300          * has completed its operation on the struct se_cmd.
5301          */
5302         if (atomic_read(&cmd->t_task.transport_lun_stop)) {
5303
5304                 DEBUG_TRANSPORT_S("wait_for_tasks: Stopping"
5305                         " wait_for_completion(&cmd->t_tasktransport_lun_fe"
5306                         "_stop_comp); for ITT: 0x%08x\n",
5307                         cmd->se_tfo->get_task_tag(cmd));
5308                 /*
5309                  * There is a special case for WRITES where a FE exception +
5310                  * LUN shutdown means ConfigFS context is still sleeping on
5311                  * transport_lun_stop_comp in transport_lun_wait_for_tasks().
5312                  * We go ahead and up transport_lun_stop_comp just to be sure
5313                  * here.
5314                  */
5315                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
5316                 complete(&cmd->t_task.transport_lun_stop_comp);
5317                 wait_for_completion(&cmd->t_task.transport_lun_fe_stop_comp);
5318                 spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
5319
5320                 transport_all_task_dev_remove_state(cmd);
5321                 /*
5322                  * At this point, the frontend who was the originator of this
5323                  * struct se_cmd, now owns the structure and can be released through
5324                  * normal means below.
5325                  */
5326                 DEBUG_TRANSPORT_S("wait_for_tasks: Stopped"
5327                         " wait_for_completion(&cmd->t_tasktransport_lun_fe_"
5328                         "stop_comp); for ITT: 0x%08x\n",
5329                         cmd->se_tfo->get_task_tag(cmd));
5330
5331                 atomic_set(&cmd->t_task.transport_lun_stop, 0);
5332         }
5333         if (!atomic_read(&cmd->t_task.t_transport_active) ||
5334              atomic_read(&cmd->t_task.t_transport_aborted))
5335                 goto remove;
5336
5337         atomic_set(&cmd->t_task.t_transport_stop, 1);
5338
5339         DEBUG_TRANSPORT_S("wait_for_tasks: Stopping %p ITT: 0x%08x"
5340                 " i_state: %d, t_state/def_t_state: %d/%d, t_transport_stop"
5341                 " = TRUE\n", cmd, cmd->se_tfo->get_task_tag(cmd),
5342                 cmd->se_tfo->get_cmd_state(cmd), cmd->t_state,
5343                 cmd->deferred_t_state);
5344
5345         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
5346
5347         wake_up_interruptible(&cmd->se_dev->dev_queue_obj.thread_wq);
5348
5349         wait_for_completion(&cmd->t_task.t_transport_stop_comp);
5350
5351         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
5352         atomic_set(&cmd->t_task.t_transport_active, 0);
5353         atomic_set(&cmd->t_task.t_transport_stop, 0);
5354
5355         DEBUG_TRANSPORT_S("wait_for_tasks: Stopped wait_for_compltion("
5356                 "&cmd->t_task.t_transport_stop_comp) for ITT: 0x%08x\n",
5357                 cmd->se_tfo->get_task_tag(cmd));
5358 remove:
5359         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
5360         if (!remove_cmd)
5361                 return;
5362
5363         transport_generic_free_cmd(cmd, 0, 0, session_reinstatement);
5364 }
5365
5366 static int transport_get_sense_codes(
5367         struct se_cmd *cmd,
5368         u8 *asc,
5369         u8 *ascq)
5370 {
5371         *asc = cmd->scsi_asc;
5372         *ascq = cmd->scsi_ascq;
5373
5374         return 0;
5375 }
5376
5377 static int transport_set_sense_codes(
5378         struct se_cmd *cmd,
5379         u8 asc,
5380         u8 ascq)
5381 {
5382         cmd->scsi_asc = asc;
5383         cmd->scsi_ascq = ascq;
5384
5385         return 0;
5386 }
5387
5388 int transport_send_check_condition_and_sense(
5389         struct se_cmd *cmd,
5390         u8 reason,
5391         int from_transport)
5392 {
5393         unsigned char *buffer = cmd->sense_buffer;
5394         unsigned long flags;
5395         int offset;
5396         u8 asc = 0, ascq = 0;
5397
5398         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
5399         if (cmd->se_cmd_flags & SCF_SENT_CHECK_CONDITION) {
5400                 spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
5401                 return 0;
5402         }
5403         cmd->se_cmd_flags |= SCF_SENT_CHECK_CONDITION;
5404         spin_unlock_irqrestore(&cmd->t_task.t_state_lock, flags);
5405
5406         if (!reason && from_transport)
5407                 goto after_reason;
5408
5409         if (!from_transport)
5410                 cmd->se_cmd_flags |= SCF_EMULATED_TASK_SENSE;
5411         /*
5412          * Data Segment and SenseLength of the fabric response PDU.
5413          *
5414          * TRANSPORT_SENSE_BUFFER is now set to SCSI_SENSE_BUFFERSIZE
5415          * from include/scsi/scsi_cmnd.h
5416          */
5417         offset = cmd->se_tfo->set_fabric_sense_len(cmd,
5418                                 TRANSPORT_SENSE_BUFFER);
5419         /*
5420          * Actual SENSE DATA, see SPC-3 7.23.2  SPC_SENSE_KEY_OFFSET uses
5421          * SENSE KEY values from include/scsi/scsi.h
5422          */
5423         switch (reason) {
5424         case TCM_NON_EXISTENT_LUN:
5425         case TCM_UNSUPPORTED_SCSI_OPCODE:
5426         case TCM_SECTOR_COUNT_TOO_MANY:
5427                 /* CURRENT ERROR */
5428                 buffer[offset] = 0x70;
5429                 /* ILLEGAL REQUEST */
5430                 buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
5431                 /* INVALID COMMAND OPERATION CODE */
5432                 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x20;
5433                 break;
5434         case TCM_UNKNOWN_MODE_PAGE:
5435                 /* CURRENT ERROR */
5436                 buffer[offset] = 0x70;
5437                 /* ILLEGAL REQUEST */
5438                 buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
5439                 /* INVALID FIELD IN CDB */
5440                 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x24;
5441                 break;
5442         case TCM_CHECK_CONDITION_ABORT_CMD:
5443                 /* CURRENT ERROR */
5444                 buffer[offset] = 0x70;
5445                 /* ABORTED COMMAND */
5446                 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
5447                 /* BUS DEVICE RESET FUNCTION OCCURRED */
5448                 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x29;
5449                 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x03;
5450                 break;
5451         case TCM_INCORRECT_AMOUNT_OF_DATA:
5452                 /* CURRENT ERROR */
5453                 buffer[offset] = 0x70;
5454                 /* ABORTED COMMAND */
5455                 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
5456                 /* WRITE ERROR */
5457                 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x0c;
5458                 /* NOT ENOUGH UNSOLICITED DATA */
5459                 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x0d;
5460                 break;
5461         case TCM_INVALID_CDB_FIELD:
5462                 /* CURRENT ERROR */
5463                 buffer[offset] = 0x70;
5464                 /* ABORTED COMMAND */
5465                 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
5466                 /* INVALID FIELD IN CDB */
5467                 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x24;
5468                 break;
5469         case TCM_INVALID_PARAMETER_LIST:
5470                 /* CURRENT ERROR */
5471                 buffer[offset] = 0x70;
5472                 /* ABORTED COMMAND */
5473                 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
5474                 /* INVALID FIELD IN PARAMETER LIST */
5475                 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x26;
5476                 break;
5477         case TCM_UNEXPECTED_UNSOLICITED_DATA:
5478                 /* CURRENT ERROR */
5479                 buffer[offset] = 0x70;
5480                 /* ABORTED COMMAND */
5481                 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
5482                 /* WRITE ERROR */
5483                 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x0c;
5484                 /* UNEXPECTED_UNSOLICITED_DATA */
5485                 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x0c;
5486                 break;
5487         case TCM_SERVICE_CRC_ERROR:
5488                 /* CURRENT ERROR */
5489                 buffer[offset] = 0x70;
5490                 /* ABORTED COMMAND */
5491                 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
5492                 /* PROTOCOL SERVICE CRC ERROR */
5493                 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x47;
5494                 /* N/A */
5495                 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x05;
5496                 break;
5497         case TCM_SNACK_REJECTED:
5498                 /* CURRENT ERROR */
5499                 buffer[offset] = 0x70;
5500                 /* ABORTED COMMAND */
5501                 buffer[offset+SPC_SENSE_KEY_OFFSET] = ABORTED_COMMAND;
5502                 /* READ ERROR */
5503                 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x11;
5504                 /* FAILED RETRANSMISSION REQUEST */
5505                 buffer[offset+SPC_ASCQ_KEY_OFFSET] = 0x13;
5506                 break;
5507         case TCM_WRITE_PROTECTED:
5508                 /* CURRENT ERROR */
5509                 buffer[offset] = 0x70;
5510                 /* DATA PROTECT */
5511                 buffer[offset+SPC_SENSE_KEY_OFFSET] = DATA_PROTECT;
5512                 /* WRITE PROTECTED */
5513                 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x27;
5514                 break;
5515         case TCM_CHECK_CONDITION_UNIT_ATTENTION:
5516                 /* CURRENT ERROR */
5517                 buffer[offset] = 0x70;
5518                 /* UNIT ATTENTION */
5519                 buffer[offset+SPC_SENSE_KEY_OFFSET] = UNIT_ATTENTION;
5520                 core_scsi3_ua_for_check_condition(cmd, &asc, &ascq);
5521                 buffer[offset+SPC_ASC_KEY_OFFSET] = asc;
5522                 buffer[offset+SPC_ASCQ_KEY_OFFSET] = ascq;
5523                 break;
5524         case TCM_CHECK_CONDITION_NOT_READY:
5525                 /* CURRENT ERROR */
5526                 buffer[offset] = 0x70;
5527                 /* Not Ready */
5528                 buffer[offset+SPC_SENSE_KEY_OFFSET] = NOT_READY;
5529                 transport_get_sense_codes(cmd, &asc, &ascq);
5530                 buffer[offset+SPC_ASC_KEY_OFFSET] = asc;
5531                 buffer[offset+SPC_ASCQ_KEY_OFFSET] = ascq;
5532                 break;
5533         case TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE:
5534         default:
5535                 /* CURRENT ERROR */
5536                 buffer[offset] = 0x70;
5537                 /* ILLEGAL REQUEST */
5538                 buffer[offset+SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST;
5539                 /* LOGICAL UNIT COMMUNICATION FAILURE */
5540                 buffer[offset+SPC_ASC_KEY_OFFSET] = 0x80;
5541                 break;
5542         }
5543         /*
5544          * This code uses linux/include/scsi/scsi.h SAM status codes!
5545          */
5546         cmd->scsi_status = SAM_STAT_CHECK_CONDITION;
5547         /*
5548          * Automatically padded, this value is encoded in the fabric's
5549          * data_length response PDU containing the SCSI defined sense data.
5550          */
5551         cmd->scsi_sense_length  = TRANSPORT_SENSE_BUFFER + offset;
5552
5553 after_reason:
5554         cmd->se_tfo->queue_status(cmd);
5555         return 0;
5556 }
5557 EXPORT_SYMBOL(transport_send_check_condition_and_sense);
5558
5559 int transport_check_aborted_status(struct se_cmd *cmd, int send_status)
5560 {
5561         int ret = 0;
5562
5563         if (atomic_read(&cmd->t_task.t_transport_aborted) != 0) {
5564                 if (!(send_status) ||
5565                      (cmd->se_cmd_flags & SCF_SENT_DELAYED_TAS))
5566                         return 1;
5567 #if 0
5568                 printk(KERN_INFO "Sending delayed SAM_STAT_TASK_ABORTED"
5569                         " status for CDB: 0x%02x ITT: 0x%08x\n",
5570                         cmd->t_task.t_task_cdb[0],
5571                         cmd->se_tfo->get_task_tag(cmd));
5572 #endif
5573                 cmd->se_cmd_flags |= SCF_SENT_DELAYED_TAS;
5574                 cmd->se_tfo->queue_status(cmd);
5575                 ret = 1;
5576         }
5577         return ret;
5578 }
5579 EXPORT_SYMBOL(transport_check_aborted_status);
5580
5581 void transport_send_task_abort(struct se_cmd *cmd)
5582 {
5583         /*
5584          * If there are still expected incoming fabric WRITEs, we wait
5585          * until until they have completed before sending a TASK_ABORTED
5586          * response.  This response with TASK_ABORTED status will be
5587          * queued back to fabric module by transport_check_aborted_status().
5588          */
5589         if (cmd->data_direction == DMA_TO_DEVICE) {
5590                 if (cmd->se_tfo->write_pending_status(cmd) != 0) {
5591                         atomic_inc(&cmd->t_task.t_transport_aborted);
5592                         smp_mb__after_atomic_inc();
5593                         cmd->scsi_status = SAM_STAT_TASK_ABORTED;
5594                         transport_new_cmd_failure(cmd);
5595                         return;
5596                 }
5597         }
5598         cmd->scsi_status = SAM_STAT_TASK_ABORTED;
5599 #if 0
5600         printk(KERN_INFO "Setting SAM_STAT_TASK_ABORTED status for CDB: 0x%02x,"
5601                 " ITT: 0x%08x\n", cmd->t_task.t_task_cdb[0],
5602                 cmd->se_tfo->get_task_tag(cmd));
5603 #endif
5604         cmd->se_tfo->queue_status(cmd);
5605 }
5606
5607 /*      transport_generic_do_tmr():
5608  *
5609  *
5610  */
5611 int transport_generic_do_tmr(struct se_cmd *cmd)
5612 {
5613         struct se_cmd *ref_cmd;
5614         struct se_device *dev = cmd->se_dev;
5615         struct se_tmr_req *tmr = cmd->se_tmr_req;
5616         int ret;
5617
5618         switch (tmr->function) {
5619         case TMR_ABORT_TASK:
5620                 ref_cmd = tmr->ref_cmd;
5621                 tmr->response = TMR_FUNCTION_REJECTED;
5622                 break;
5623         case TMR_ABORT_TASK_SET:
5624         case TMR_CLEAR_ACA:
5625         case TMR_CLEAR_TASK_SET:
5626                 tmr->response = TMR_TASK_MGMT_FUNCTION_NOT_SUPPORTED;
5627                 break;
5628         case TMR_LUN_RESET:
5629                 ret = core_tmr_lun_reset(dev, tmr, NULL, NULL);
5630                 tmr->response = (!ret) ? TMR_FUNCTION_COMPLETE :
5631                                          TMR_FUNCTION_REJECTED;
5632                 break;
5633         case TMR_TARGET_WARM_RESET:
5634                 tmr->response = TMR_FUNCTION_REJECTED;
5635                 break;
5636         case TMR_TARGET_COLD_RESET:
5637                 tmr->response = TMR_FUNCTION_REJECTED;
5638                 break;
5639         default:
5640                 printk(KERN_ERR "Uknown TMR function: 0x%02x.\n",
5641                                 tmr->function);
5642                 tmr->response = TMR_FUNCTION_REJECTED;
5643                 break;
5644         }
5645
5646         cmd->t_state = TRANSPORT_ISTATE_PROCESSING;
5647         cmd->se_tfo->queue_tm_rsp(cmd);
5648
5649         transport_cmd_check_stop(cmd, 2, 0);
5650         return 0;
5651 }
5652
5653 /*
5654  *      Called with spin_lock_irq(&dev->execute_task_lock); held
5655  *
5656  */
5657 static struct se_task *
5658 transport_get_task_from_state_list(struct se_device *dev)
5659 {
5660         struct se_task *task;
5661
5662         if (list_empty(&dev->state_task_list))
5663                 return NULL;
5664
5665         list_for_each_entry(task, &dev->state_task_list, t_state_list)
5666                 break;
5667
5668         list_del(&task->t_state_list);
5669         atomic_set(&task->task_state_active, 0);
5670
5671         return task;
5672 }
5673
5674 static void transport_processing_shutdown(struct se_device *dev)
5675 {
5676         struct se_cmd *cmd;
5677         struct se_task *task;
5678         unsigned long flags;
5679         /*
5680          * Empty the struct se_device's struct se_task state list.
5681          */
5682         spin_lock_irqsave(&dev->execute_task_lock, flags);
5683         while ((task = transport_get_task_from_state_list(dev))) {
5684                 if (!task->task_se_cmd) {
5685                         printk(KERN_ERR "task->task_se_cmd is NULL!\n");
5686                         continue;
5687                 }
5688                 cmd = task->task_se_cmd;
5689
5690                 spin_unlock_irqrestore(&dev->execute_task_lock, flags);
5691
5692                 spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
5693
5694                 DEBUG_DO("PT: cmd: %p task: %p ITT/CmdSN: 0x%08x/0x%08x,"
5695                         " i_state/def_i_state: %d/%d, t_state/def_t_state:"
5696                         " %d/%d cdb: 0x%02x\n", cmd, task,
5697                         cmd->se_tfo->get_task_tag(cmd), cmd->cmd_sn,
5698                         cmd->se_tfo->get_cmd_state(cmd), cmd->deferred_i_state,
5699                         cmd->t_state, cmd->deferred_t_state,
5700                         cmd->t_task.t_task_cdb[0]);
5701                 DEBUG_DO("PT: ITT[0x%08x] - t_task_cdbs: %d t_task_cdbs_left:"
5702                         " %d t_task_cdbs_sent: %d -- t_transport_active: %d"
5703                         " t_transport_stop: %d t_transport_sent: %d\n",
5704                         cmd->se_tfo->get_task_tag(cmd),
5705                         cmd->t_task.t_task_cdbs,
5706                         atomic_read(&cmd->t_task.t_task_cdbs_left),
5707                         atomic_read(&cmd->t_task.t_task_cdbs_sent),
5708                         atomic_read(&cmd->t_task.t_transport_active),
5709                         atomic_read(&cmd->t_task.t_transport_stop),
5710                         atomic_read(&cmd->t_task.t_transport_sent));
5711
5712                 if (atomic_read(&task->task_active)) {
5713                         atomic_set(&task->task_stop, 1);
5714                         spin_unlock_irqrestore(
5715                                 &cmd->t_task.t_state_lock, flags);
5716
5717                         DEBUG_DO("Waiting for task: %p to shutdown for dev:"
5718                                 " %p\n", task, dev);
5719                         wait_for_completion(&task->task_stop_comp);
5720                         DEBUG_DO("Completed task: %p shutdown for dev: %p\n",
5721                                 task, dev);
5722
5723                         spin_lock_irqsave(&cmd->t_task.t_state_lock, flags);
5724                         atomic_dec(&cmd->t_task.t_task_cdbs_left);
5725
5726                         atomic_set(&task->task_active, 0);
5727                         atomic_set(&task->task_stop, 0);
5728                 } else {
5729                         if (atomic_read(&task->task_execute_queue) != 0)
5730                                 transport_remove_task_from_execute_queue(task, dev);
5731                 }
5732                 __transport_stop_task_timer(task, &flags);
5733
5734                 if (!(atomic_dec_and_test(&cmd->t_task.t_task_cdbs_ex_left))) {
5735                         spin_unlock_irqrestore(
5736                                         &cmd->t_task.t_state_lock, flags);
5737
5738                         DEBUG_DO("Skipping task: %p, dev: %p for"
5739                                 " t_task_cdbs_ex_left: %d\n", task, dev,
5740                                 atomic_read(&cmd->t_task.t_task_cdbs_ex_left));
5741
5742                         spin_lock_irqsave(&dev->execute_task_lock, flags);
5743                         continue;
5744                 }
5745
5746                 if (atomic_read(&cmd->t_task.t_transport_active)) {
5747                         DEBUG_DO("got t_transport_active = 1 for task: %p, dev:"
5748                                         " %p\n", task, dev);
5749
5750                         if (atomic_read(&cmd->t_task.t_fe_count)) {
5751                                 spin_unlock_irqrestore(
5752                                         &cmd->t_task.t_state_lock, flags);
5753                                 transport_send_check_condition_and_sense(
5754                                         cmd, TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE,
5755                                         0);
5756                                 transport_remove_cmd_from_queue(cmd,
5757                                         &cmd->se_dev->dev_queue_obj);
5758
5759                                 transport_lun_remove_cmd(cmd);
5760                                 transport_cmd_check_stop(cmd, 1, 0);
5761                         } else {
5762                                 spin_unlock_irqrestore(
5763                                         &cmd->t_task.t_state_lock, flags);
5764
5765                                 transport_remove_cmd_from_queue(cmd,
5766                                         &cmd->se_dev->dev_queue_obj);
5767
5768                                 transport_lun_remove_cmd(cmd);
5769
5770                                 if (transport_cmd_check_stop(cmd, 1, 0))
5771                                         transport_generic_remove(cmd, 0, 0);
5772                         }
5773
5774                         spin_lock_irqsave(&dev->execute_task_lock, flags);
5775                         continue;
5776                 }
5777                 DEBUG_DO("Got t_transport_active = 0 for task: %p, dev: %p\n",
5778                                 task, dev);
5779
5780                 if (atomic_read(&cmd->t_task.t_fe_count)) {
5781                         spin_unlock_irqrestore(
5782                                 &cmd->t_task.t_state_lock, flags);
5783                         transport_send_check_condition_and_sense(cmd,
5784                                 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE, 0);
5785                         transport_remove_cmd_from_queue(cmd,
5786                                 &cmd->se_dev->dev_queue_obj);
5787
5788                         transport_lun_remove_cmd(cmd);
5789                         transport_cmd_check_stop(cmd, 1, 0);
5790                 } else {
5791                         spin_unlock_irqrestore(
5792                                 &cmd->t_task.t_state_lock, flags);
5793
5794                         transport_remove_cmd_from_queue(cmd,
5795                                 &cmd->se_dev->dev_queue_obj);
5796                         transport_lun_remove_cmd(cmd);
5797
5798                         if (transport_cmd_check_stop(cmd, 1, 0))
5799                                 transport_generic_remove(cmd, 0, 0);
5800                 }
5801
5802                 spin_lock_irqsave(&dev->execute_task_lock, flags);
5803         }
5804         spin_unlock_irqrestore(&dev->execute_task_lock, flags);
5805         /*
5806          * Empty the struct se_device's struct se_cmd list.
5807          */
5808         while ((cmd = transport_get_cmd_from_queue(&dev->dev_queue_obj))) {
5809
5810                 DEBUG_DO("From Device Queue: cmd: %p t_state: %d\n",
5811                                 cmd, cmd->t_state);
5812
5813                 if (atomic_read(&cmd->t_task.t_fe_count)) {
5814                         transport_send_check_condition_and_sense(cmd,
5815                                 TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE, 0);
5816
5817                         transport_lun_remove_cmd(cmd);
5818                         transport_cmd_check_stop(cmd, 1, 0);
5819                 } else {
5820                         transport_lun_remove_cmd(cmd);
5821                         if (transport_cmd_check_stop(cmd, 1, 0))
5822                                 transport_generic_remove(cmd, 0, 0);
5823                 }
5824         }
5825 }
5826
5827 /*      transport_processing_thread():
5828  *
5829  *
5830  */
5831 static int transport_processing_thread(void *param)
5832 {
5833         int ret;
5834         struct se_cmd *cmd;
5835         struct se_device *dev = (struct se_device *) param;
5836
5837         set_user_nice(current, -20);
5838
5839         while (!kthread_should_stop()) {
5840                 ret = wait_event_interruptible(dev->dev_queue_obj.thread_wq,
5841                                 atomic_read(&dev->dev_queue_obj.queue_cnt) ||
5842                                 kthread_should_stop());
5843                 if (ret < 0)
5844                         goto out;
5845
5846                 spin_lock_irq(&dev->dev_status_lock);
5847                 if (dev->dev_status & TRANSPORT_DEVICE_SHUTDOWN) {
5848                         spin_unlock_irq(&dev->dev_status_lock);
5849                         transport_processing_shutdown(dev);
5850                         continue;
5851                 }
5852                 spin_unlock_irq(&dev->dev_status_lock);
5853
5854 get_cmd:
5855                 __transport_execute_tasks(dev);
5856
5857                 cmd = transport_get_cmd_from_queue(&dev->dev_queue_obj);
5858                 if (!cmd)
5859                         continue;
5860
5861                 switch (cmd->t_state) {
5862                 case TRANSPORT_NEW_CMD_MAP:
5863                         if (!(cmd->se_tfo->new_cmd_map)) {
5864                                 printk(KERN_ERR "cmd->se_tfo->new_cmd_map is"
5865                                         " NULL for TRANSPORT_NEW_CMD_MAP\n");
5866                                 BUG();
5867                         }
5868                         ret = cmd->se_tfo->new_cmd_map(cmd);
5869                         if (ret < 0) {
5870                                 cmd->transport_error_status = ret;
5871                                 transport_generic_request_failure(cmd, NULL,
5872                                                 0, (cmd->data_direction !=
5873                                                     DMA_TO_DEVICE));
5874                                 break;
5875                         }
5876                         /* Fall through */
5877                 case TRANSPORT_NEW_CMD:
5878                         ret = transport_generic_new_cmd(cmd);
5879                         if (ret < 0) {
5880                                 cmd->transport_error_status = ret;
5881                                 transport_generic_request_failure(cmd, NULL,
5882                                         0, (cmd->data_direction !=
5883                                          DMA_TO_DEVICE));
5884                         }
5885                         break;
5886                 case TRANSPORT_PROCESS_WRITE:
5887                         transport_generic_process_write(cmd);
5888                         break;
5889                 case TRANSPORT_COMPLETE_OK:
5890                         transport_stop_all_task_timers(cmd);
5891                         transport_generic_complete_ok(cmd);
5892                         break;
5893                 case TRANSPORT_REMOVE:
5894                         transport_generic_remove(cmd, 1, 0);
5895                         break;
5896                 case TRANSPORT_FREE_CMD_INTR:
5897                         transport_generic_free_cmd(cmd, 0, 1, 0);
5898                         break;
5899                 case TRANSPORT_PROCESS_TMR:
5900                         transport_generic_do_tmr(cmd);
5901                         break;
5902                 case TRANSPORT_COMPLETE_FAILURE:
5903                         transport_generic_request_failure(cmd, NULL, 1, 1);
5904                         break;
5905                 case TRANSPORT_COMPLETE_TIMEOUT:
5906                         transport_stop_all_task_timers(cmd);
5907                         transport_generic_request_timeout(cmd);
5908                         break;
5909                 default:
5910                         printk(KERN_ERR "Unknown t_state: %d deferred_t_state:"
5911                                 " %d for ITT: 0x%08x i_state: %d on SE LUN:"
5912                                 " %u\n", cmd->t_state, cmd->deferred_t_state,
5913                                 cmd->se_tfo->get_task_tag(cmd),
5914                                 cmd->se_tfo->get_cmd_state(cmd),
5915                                 cmd->se_lun->unpacked_lun);
5916                         BUG();
5917                 }
5918
5919                 goto get_cmd;
5920         }
5921
5922 out:
5923         transport_release_all_cmds(dev);
5924         dev->process_thread = NULL;
5925         return 0;
5926 }