ibmvscsis: Enable Logical Partition Migration Support

Changes to support a new mechanism from phyp to better synchronize the
logical partition migration (LPM) of the client partition.
This includes a new VIOCTL to register that we support this new
functionality, and 2 new Transport Event types, and finally another
new VIOCTL to let phyp know once we're ready for the Suspend.

Signed-off-by: Michael Cyr <mikecyr@us.ibm.com>
Signed-off-by: Bryant G. Ly <bryantly@linux.vnet.ibm.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>

diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
index abf6026..3571052 100644 (file)
--- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
+++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.c
@@ -155,6 +155,9 @@ static long ibmvscsis_unregister_command_q(struct scsi_info *vscsi)
                qrc = h_free_crq(vscsi->dds.unit_id);
                switch (qrc) {
                case H_SUCCESS:
+                       spin_lock_bh(&vscsi->intr_lock);
+                       vscsi->flags &= ~PREP_FOR_SUSPEND_FLAGS;
+                       spin_unlock_bh(&vscsi->intr_lock);
                        break;
 
                case H_HARDWARE:
@@ -422,6 +425,9 @@ static void ibmvscsis_disconnect(struct work_struct *work)
        new_state = vscsi->new_state;
        vscsi->new_state = 0;
 
+       vscsi->flags |= DISCONNECT_SCHEDULED;
+       vscsi->flags &= ~SCHEDULE_DISCONNECT;
+
        pr_debug("disconnect: flags 0x%x, state 0x%hx\n", vscsi->flags,
                 vscsi->state);
 
@@ -802,6 +808,13 @@ static long ibmvscsis_establish_new_q(struct scsi_info *vscsi)
        long rc = ADAPT_SUCCESS;
        uint format;
 
+       rc = h_vioctl(vscsi->dds.unit_id, H_ENABLE_PREPARE_FOR_SUSPEND, 30000,
+                     0, 0, 0, 0);
+       if (rc == H_SUCCESS)
+               vscsi->flags |= PREP_FOR_SUSPEND_ENABLED;
+       else if (rc != H_NOT_FOUND)
+               pr_err("Error from Enable Prepare for Suspend: %ld\n", rc);
+
        vscsi->flags &= PRESERVE_FLAG_FIELDS;
        vscsi->rsp_q_timer.timer_pops = 0;
        vscsi->debit = 0;
@@ -951,6 +964,63 @@ static void ibmvscsis_free_cmd_resources(struct scsi_info *vscsi,
 }
 
 /**
+ * ibmvscsis_ready_for_suspend() - Helper function to call VIOCTL
+ * @vscsi:     Pointer to our adapter structure
+ * @idle:      Indicates whether we were called from adapter_idle.  This
+ *             is important to know if we need to do a disconnect, since if
+ *             we're called from adapter_idle, we're still processing the
+ *             current disconnect, so we can't just call post_disconnect.
+ *
+ * This function is called when the adapter is idle when phyp has sent
+ * us a Prepare for Suspend Transport Event.
+ *
+ * EXECUTION ENVIRONMENT:
+ *     Process or interrupt environment called with interrupt lock held
+ */
+static long ibmvscsis_ready_for_suspend(struct scsi_info *vscsi, bool idle)
+{
+       long rc = 0;
+       struct viosrp_crq *crq;
+
+       /* See if there is a Resume event in the queue */
+       crq = vscsi->cmd_q.base_addr + vscsi->cmd_q.index;
+
+       pr_debug("ready_suspend: flags 0x%x, state 0x%hx crq_valid:%x\n",
+                vscsi->flags, vscsi->state, (int)crq->valid);
+
+       if (!(vscsi->flags & PREP_FOR_SUSPEND_ABORTED) && !(crq->valid)) {
+               rc = h_vioctl(vscsi->dds.unit_id, H_READY_FOR_SUSPEND, 0, 0, 0,
+                             0, 0);
+               if (rc) {
+                       pr_err("Ready for Suspend Vioctl failed: %ld\n", rc);
+                       rc = 0;
+               }
+       } else if (((vscsi->flags & PREP_FOR_SUSPEND_OVERWRITE) &&
+                   (vscsi->flags & PREP_FOR_SUSPEND_ABORTED)) ||
+                  ((crq->valid) && ((crq->valid != VALID_TRANS_EVENT) ||
+                                    (crq->format != RESUME_FROM_SUSP)))) {
+               if (idle) {
+                       vscsi->state = ERR_DISCONNECT_RECONNECT;
+                       ibmvscsis_reset_queue(vscsi);
+                       rc = -1;
+               } else if (vscsi->state == CONNECTED) {
+                       ibmvscsis_post_disconnect(vscsi,
+                                                 ERR_DISCONNECT_RECONNECT, 0);
+               }
+
+               vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE;
+
+               if ((crq->valid) && ((crq->valid != VALID_TRANS_EVENT) ||
+                                    (crq->format != RESUME_FROM_SUSP)))
+                       pr_err("Invalid element in CRQ after Prepare for Suspend");
+       }
+
+       vscsi->flags &= ~(PREP_FOR_SUSPEND_PENDING | PREP_FOR_SUSPEND_ABORTED);
+
+       return rc;
+}
+
+/**
  * ibmvscsis_trans_event() - Handle a Transport Event
  * @vscsi:     Pointer to our adapter structure
  * @crq:       Pointer to CRQ entry containing the Transport Event
@@ -974,18 +1044,8 @@ static long ibmvscsis_trans_event(struct scsi_info *vscsi,
        case PARTNER_FAILED:
        case PARTNER_DEREGISTER:
                ibmvscsis_delete_client_info(vscsi, true);
-               break;
-
-       default:
-               rc = ERROR;
-               dev_err(&vscsi->dev, "trans_event: invalid format %d\n",
-                       (uint)crq->format);
-               ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT,
-                                         RESPONSE_Q_DOWN);
-               break;
-       }
-
-       if (rc == ADAPT_SUCCESS) {
+               if (crq->format == MIGRATED)
+                       vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE;
                switch (vscsi->state) {
                case NO_QUEUE:
                case ERR_DISCONNECTED:
@@ -1034,6 +1094,60 @@ static long ibmvscsis_trans_event(struct scsi_info *vscsi,
                        vscsi->flags |= (RESPONSE_Q_DOWN | TRANS_EVENT);
                        break;
                }
+               break;
+
+       case PREPARE_FOR_SUSPEND:
+               pr_debug("Prep for Suspend, crq status = 0x%x\n",
+                        (int)crq->status);
+               switch (vscsi->state) {
+               case ERR_DISCONNECTED:
+               case WAIT_CONNECTION:
+               case CONNECTED:
+                       ibmvscsis_ready_for_suspend(vscsi, false);
+                       break;
+               case SRP_PROCESSING:
+                       vscsi->resume_state = vscsi->state;
+                       vscsi->flags |= PREP_FOR_SUSPEND_PENDING;
+                       if (crq->status == CRQ_ENTRY_OVERWRITTEN)
+                               vscsi->flags |= PREP_FOR_SUSPEND_OVERWRITE;
+                       ibmvscsis_post_disconnect(vscsi, WAIT_IDLE, 0);
+                       break;
+               case NO_QUEUE:
+               case UNDEFINED:
+               case UNCONFIGURING:
+               case WAIT_ENABLED:
+               case ERR_DISCONNECT:
+               case ERR_DISCONNECT_RECONNECT:
+               case WAIT_IDLE:
+                       pr_err("Invalid state for Prepare for Suspend Trans Event: 0x%x\n",
+                              vscsi->state);
+                       break;
+               }
+               break;
+
+       case RESUME_FROM_SUSP:
+               pr_debug("Resume from Suspend, crq status = 0x%x\n",
+                        (int)crq->status);
+               if (vscsi->flags & PREP_FOR_SUSPEND_PENDING) {
+                       vscsi->flags |= PREP_FOR_SUSPEND_ABORTED;
+               } else {
+                       if ((crq->status == CRQ_ENTRY_OVERWRITTEN) ||
+                           (vscsi->flags & PREP_FOR_SUSPEND_OVERWRITE)) {
+                               ibmvscsis_post_disconnect(vscsi,
+                                                         ERR_DISCONNECT_RECONNECT,
+                                                         0);
+                               vscsi->flags &= ~PREP_FOR_SUSPEND_OVERWRITE;
+                       }
+               }
+               break;
+
+       default:
+               rc = ERROR;
+               dev_err(&vscsi->dev, "trans_event: invalid format %d\n",
+                       (uint)crq->format);
+               ibmvscsis_post_disconnect(vscsi, ERR_DISCONNECT,
+                                         RESPONSE_Q_DOWN);
+               break;
        }
 
        rc = vscsi->flags & SCHEDULE_DISCONNECT;
@@ -1201,6 +1315,7 @@ static struct ibmvscsis_cmd *ibmvscsis_get_free_cmd(struct scsi_info *vscsi)
 static void ibmvscsis_adapter_idle(struct scsi_info *vscsi)
 {
        int free_qs = false;
+       long rc = 0;
 
        pr_debug("adapter_idle: flags 0x%x, state 0x%hx\n", vscsi->flags,
                 vscsi->state);
@@ -1240,7 +1355,14 @@ static void ibmvscsis_adapter_idle(struct scsi_info *vscsi)
                vscsi->rsp_q_timer.timer_pops = 0;
                vscsi->debit = 0;
                vscsi->credit = 0;
-               if (vscsi->flags & TRANS_EVENT) {
+               if (vscsi->flags & PREP_FOR_SUSPEND_PENDING) {
+                       vscsi->state = vscsi->resume_state;
+                       vscsi->resume_state = 0;
+                       rc = ibmvscsis_ready_for_suspend(vscsi, true);
+                       vscsi->flags &= ~DISCONNECT_SCHEDULED;
+                       if (rc)
+                               break;
+               } else if (vscsi->flags & TRANS_EVENT) {
                        vscsi->state = WAIT_CONNECTION;
                        vscsi->flags &= PRESERVE_FLAG_FIELDS;
                } else {

diff --git a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h
index b4391a8..cc96c27 100644 (file)
--- a/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h
+++ b/drivers/scsi/ibmvscsi_tgt/ibmvscsi_tgt.h
@@ -262,6 +262,14 @@ struct scsi_info {
 #define DISCONNECT_SCHEDULED          0x00800
        /* remove function is sleeping */
 #define CFG_SLEEPING                  0x01000
+       /* Register for Prepare for Suspend Transport Events */
+#define PREP_FOR_SUSPEND_ENABLED      0x02000
+       /* Prepare for Suspend event sent */
+#define PREP_FOR_SUSPEND_PENDING      0x04000
+       /* Resume from Suspend event sent */
+#define PREP_FOR_SUSPEND_ABORTED      0x08000
+       /* Prepare for Suspend event overwrote another CRQ entry */
+#define PREP_FOR_SUSPEND_OVERWRITE    0x10000
        u32 flags;
        /* adapter lock */
        spinlock_t intr_lock;
@@ -272,6 +280,7 @@ struct scsi_info {
        /* used in crq, to tag what iu the response is for */
        u64  empty_iu_tag;
        uint new_state;
+       uint resume_state;
        /* control block for the response queue timer */
        struct timer_cb rsp_q_timer;
        /* keep last client to enable proper accounting */
@@ -324,8 +333,13 @@ struct scsi_info {
 #define TARGET_STOP(VSCSI) (long)(((VSCSI)->state & DONT_PROCESS_STATE) | \
                                  ((VSCSI)->flags & BLOCK))
 
+#define PREP_FOR_SUSPEND_FLAGS  (PREP_FOR_SUSPEND_ENABLED | \
+                                PREP_FOR_SUSPEND_PENDING | \
+                                PREP_FOR_SUSPEND_ABORTED | \
+                                PREP_FOR_SUSPEND_OVERWRITE)
+
 /* flag bit that are not reset during disconnect */
-#define PRESERVE_FLAG_FIELDS 0
+#define PRESERVE_FLAG_FIELDS (PREP_FOR_SUSPEND_FLAGS)
 
 #define vio_iu(IUE) ((union viosrp_iu *)((IUE)->sbuf->buf))
 
@@ -333,8 +347,15 @@ struct scsi_info {
 #define WRITE_CMD(cdb) (((cdb)[0] & 0x1F) == 0xA)
 
 #ifndef H_GET_PARTNER_INFO
-#define H_GET_PARTNER_INFO      0x0000000000000008LL
+#define H_GET_PARTNER_INFO              0x0000000000000008LL
+#endif
+#ifndef H_ENABLE_PREPARE_FOR_SUSPEND
+#define H_ENABLE_PREPARE_FOR_SUSPEND    0x000000000000001DLL
 #endif
+#ifndef H_READY_FOR_SUSPEND
+#define H_READY_FOR_SUSPEND             0x000000000000001ELL
+#endif
+
 
 #define h_copy_rdma(l, sa, sb, da, db) \
                plpar_hcall_norets(H_COPY_RDMA, l, sa, sb, da, db)

diff --git a/drivers/scsi/ibmvscsi_tgt/libsrp.h b/drivers/scsi/ibmvscsi_tgt/libsrp.h
index 4696f33..9fec55b 100644 (file)
--- a/drivers/scsi/ibmvscsi_tgt/libsrp.h
+++ b/drivers/scsi/ibmvscsi_tgt/libsrp.h
@@ -30,10 +30,13 @@ enum srp_trans_event {
        UNUSED_FORMAT = 0,
        PARTNER_FAILED = 1,
        PARTNER_DEREGISTER = 2,
-       MIGRATED = 6
+       MIGRATED = 6,
+       PREPARE_FOR_SUSPEND = 9,
+       RESUME_FROM_SUSP = 0xA
 };
 
 enum srp_status {
+       CRQ_ENTRY_OVERWRITTEN = 0x20,
        HEADER_DESCRIPTOR = 0xF1,
        PING = 0xF5,
        PING_RESPONSE = 0xF6